diff --git a/src/ucp/core/ucp_ep.c b/src/ucp/core/ucp_ep.c index f185398e6a7..1d287e065ab 100644 --- a/src/ucp/core/ucp_ep.c +++ b/src/ucp/core/ucp_ep.c @@ -2128,6 +2128,7 @@ static void ucp_ep_recovery_arg_free(ucp_ep_h ep) seen_peer_id = arg->seen_peer_id; ucp_ep_refcount_assert(ep, probe, ==, 0); + ucp_wireup_tf_reply_discard(ep); ucp_ep_tf_cleanup(ep); ucs_free(arg); ep->ext->recovery_arg = NULL; @@ -2506,8 +2507,14 @@ int ucp_ep_recovery_progress(ucp_ep_h ep) failed = ucp_ep_get_failed_lanes(ep); if (failed == 0) { - /* Recovery completed between rounds, the ep operates normally. */ - ucs_assert(ucp_ep_recovery_arg(ep) == NULL); + /* A peer request moves the queue pair to error and waits for that + * completion before this lane is failed. Recovery stays idle until + * the completion holds it. */ + if ((ucp_ep_recovery_arg(ep) == NULL) || + (ucp_ep_recovery_arg(ep)->tf_reply == NULL)) { + ucs_assert(ucp_ep_recovery_arg(ep) == NULL); + } + goto done; } diff --git a/src/ucp/core/ucp_ep.h b/src/ucp/core/ucp_ep.h index 6faec432053..ac1cc3b8c0e 100644 --- a/src/ucp/core/ucp_ep.h +++ b/src/ucp/core/ucp_ep.h @@ -564,6 +564,9 @@ typedef struct ucp_ep_recovery_arg { * outstanding operations are resolved, * outstanding re-posts also go here */ ucs_queue_head_t tf_pending_q; + /* LANES_ADDR reply waiting for error completions. Owned by wireup. + * NULL when no reply is waiting. */ + void *tf_reply; } ucp_ep_recovery_arg_t; diff --git a/src/ucp/core/ucp_worker.c b/src/ucp/core/ucp_worker.c index b674833d888..20462b1215c 100644 --- a/src/ucp/core/ucp_worker.c +++ b/src/ucp/core/ucp_worker.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -482,18 +483,22 @@ ucp_worker_iface_handle_uct_ep_failure(ucp_ep_h ucp_ep, ucp_lane_index_t lane, if (wireup_ep == NULL) { /* This endpoint is already held. Keep it until its completion. */ if (ucp_ep_tf_is_held(ucp_ep, lane, uct_ep)) { + ucp_wireup_tf_lane_errored(ucp_ep, lane); return UCS_INPROGRESS; } if (ucp_ep_tf_hold(ucp_ep, lane, uct_ep) == UCS_OK) { /* Purge after return, once the outstanding range is - * recorded. */ + * recorded. The hold is the error this request was waiting + * for. A deferred reply rebuilds the lane only after it. */ ucp_ep_tf_schedule_token_purge(ucp_ep, lane); + ucp_wireup_tf_lane_errored(ucp_ep, lane); return UCS_INPROGRESS; } } ucp_ep_set_lanes_failed(ucp_ep, UCS_BIT(lane), status); + ucp_wireup_tf_lane_errored(ucp_ep, lane); return UCS_OK; } @@ -565,8 +570,8 @@ ucp_worker_iface_error_handler(void *arg, uct_ep_h uct_ep, ucs_status_t status) UCS_ASYNC_BLOCK(&worker->async); - ucs_debug("worker %p: error handler called for UCT EP %p: %s", - worker, uct_ep, ucs_status_string(status)); + ucs_debug("worker %p: error handler called for UCT EP %p: %s", worker, + uct_ep, ucs_status_string(status)); if (ucp_worker_is_uct_ep_discarding(worker, uct_ep)) { ucs_debug("UCT EP %p is being discarded on UCP Worker %p", @@ -588,6 +593,9 @@ ucp_worker_iface_error_handler(void *arg, uct_ep_h uct_ep, ucs_status_t status) } } + ucs_debug("worker %p: uct_ep %p is ep %p lane %d slot %p", worker, uct_ep, + ucp_ep, lane, ucp_ep_get_lane(ucp_ep, lane)); + status = ucp_worker_iface_handle_uct_ep_failure(ucp_ep, lane, uct_ep, status); diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c index 0e7bcc8ab3a..b8177b18849 100644 --- a/src/ucp/wireup/wireup.c +++ b/src/ucp/wireup/wireup.c @@ -1264,7 +1264,8 @@ ucp_wireup_skip_token_section(ucp_lane_map_t lane_map, const void *section, static void ucp_wireup_derive_rx_tokens(ucp_ep_h ep, ucp_lane_map_t lane_map, const ucp_wireup_lane_token_t *tx_tokens, - ucp_wireup_lane_token_t *rx_tokens) + ucp_wireup_lane_token_t *rx_tokens, + int require_open) { ucp_lane_index_t lane; ucs_status_t status; @@ -1277,8 +1278,13 @@ ucp_wireup_derive_rx_tokens(ucp_ep_h ep, ucp_lane_map_t lane_map, } status = ucp_ep_tf_derive_rx(ep, lane, tx_tokens[lane].token, - tx_tokens[lane].len, 1, &rx_token, - &rx_len); + tx_tokens[lane].len, require_open, + &rx_token, &rx_len); + if (status == UCS_ERR_NO_ELEM) { + ucs_debug("ep %p: lane %d: peer qp is gone", ep, lane); + continue; + } + if (status != UCS_OK) { ucs_debug("ep %p: lane %d: rx token was not derived: %s", ep, lane, ucs_status_string(status)); @@ -1307,76 +1313,386 @@ ucp_wireup_free_lane_tokens(ucp_lane_map_t lane_map, } } -static UCS_F_NOINLINE void -ucp_wireup_process_lanes_addr_request( +/* Reply saved until every lane in waiting has an error completion. The + * address bytes are a copy of the request payload. The unpacked form points + * into the message and does not outlive that handler. */ +typedef struct { + uint32_t request_id; + ucp_lane_map_t requested_lane_map; + ucp_lane_map_t provided_lane_map; + ucp_lane_map_t skip_lanes; + ucp_lane_map_t waiting; + uint8_t scheduled; + ucp_wireup_lane_token_t rx_tokens[UCP_MAX_LANES]; + void *address; + size_t address_length; +} ucp_wireup_tf_reply_t; + + +/* recovery_arg shares storage with the close request. A sockaddr endpoint + * does not support failover, so that pointer is not recovery state. */ +static ucp_ep_recovery_arg_t *ucp_wireup_tf_recovery_arg(ucp_ep_h ep) +{ + if (!ucp_ep_err_mode_eq(ep, UCP_ERR_HANDLING_MODE_FAILOVER) || + (ucp_ep_get_cm_lane(ep) != UCP_NULL_LANE)) { + return NULL; + } + + return ucp_ep_recovery_arg(ep); +} + +static ucp_wireup_tf_reply_t *ucp_wireup_tf_reply_get(ucp_ep_h ep) +{ + ucp_ep_recovery_arg_t *arg = ucp_wireup_tf_recovery_arg(ep); + + if ((arg == NULL) || (arg->tf_reply == NULL)) { + return NULL; + } + + return arg->tf_reply; +} + +static void ucp_wireup_tf_reply_free(ucp_wireup_tf_reply_t *reply) +{ + if (reply == NULL) { + return; + } + + ucp_wireup_free_lane_tokens(reply->provided_lane_map, reply->rx_tokens); + ucs_free(reply->address); + ucs_free(reply); +} + +/* Lanes whose replacement queue pair is already stored in the wireup proxy. + * The proxy is not ready, so the data path still skips them. */ +static ucp_lane_map_t +ucp_wireup_lanes_with_next(ucp_ep_h ep, ucp_lane_map_t lanes) +{ + ucp_lane_map_t with_next = 0; + ucp_wireup_ep_t *wireup_ep; + ucp_lane_index_t lane; + + ucs_for_each_bit(lane, lanes) { + wireup_ep = ucp_wireup_ep(ucp_ep_get_lane(ep, lane)); + if ((wireup_ep != NULL) && ucp_wireup_ep_has_next_ep(wireup_ep)) { + with_next |= UCS_BIT(lane); + } + } + + return with_next; +} + +static void +ucp_wireup_lanes_addr_request_reply(ucp_ep_h ep, ucp_wireup_tf_reply_t *reply, + const ucp_unpacked_address_t *remote_address) +{ + ucp_lane_map_t peer_unaware, to_rebuild, rebuilt, provided; + ucs_status_t status; + + /* Lanes still up take the software path. A lane invalidated for this + * request is skipped here: its error completion holds it, and the hold + * puts it in the failed set that rebuild walks. */ + peer_unaware = reply->provided_lane_map & ~ucp_ep_get_failed_lanes(ep) & + ~reply->skip_lanes; + if (peer_unaware != 0) { + ucs_debug("ep %p: LANES_ADDR_REQ triggering local failover for " + "asymmetric failure on lanes 0x%" PRIx64, + ep, (uint64_t)peer_unaware); + status = ucp_ep_failover_reconfig(ep, peer_unaware, + UCS_ERR_CONNECTION_RESET); + if (status != UCS_OK) { + ucs_diag("ep %p: local failover for LANES_ADDR_REQ failed: %s", ep, + ucs_status_string(status)); + } + } + + to_rebuild = reply->provided_lane_map & ucp_ep_get_failed_lanes(ep); + rebuilt = ucp_ep_recovery_rebuild_lanes(ep, to_rebuild, remote_address); + /* Publish a lane as soon as its replacement queue pair exists. A lane + * whose aux probe is still running is not ready, and is not switched + * onto the data path. */ + provided = rebuilt | ucp_wireup_lanes_with_next(ep, to_rebuild); + ucs_debug("ep %p: LANES_ADDR_REQ requested=0x%" PRIx64 + " provided=0x%" PRIx64 " to_rebuild=0x%" PRIx64 + " rebuilt=0x%" PRIx64, + ep, (uint64_t)reply->requested_lane_map, + (uint64_t)reply->provided_lane_map, (uint64_t)to_rebuild, + (uint64_t)rebuilt); + + if (rebuilt != 0) { + ucp_wireup_eps_progress_sched(ep); + } + + /* Always reply, even if provided == 0, so the initiator knows the peer + * handled the request. The reply answers the lanes the request provided, + * so its RX tokens belong to them. */ + ucp_wireup_send_lanes_addr_msg(ep, UCP_WIREUP_MSG_LANES_ADDR_REPLY, + reply->provided_lane_map, provided, + reply->request_id, reply->rx_tokens); +} + +static unsigned ucp_wireup_tf_reply_progress(void *arg); + +static int +ucp_wireup_tf_reply_remove_filter(const ucs_callbackq_elem_t *elem, void *arg) +{ + return (elem->cb == ucp_wireup_tf_reply_progress) && (elem->arg == arg); +} + +static void ucp_wireup_tf_reply_schedule(ucp_ep_h ep) +{ + ucp_wireup_tf_reply_t *reply = ucp_wireup_tf_reply_get(ep); + + if ((reply == NULL) || (reply->waiting != 0) || reply->scheduled) { + return; + } + + /* Run after the error handler returns, so the transport has saved the + * transmit range before the reply rebuilds the lane. */ + reply->scheduled = 1; + ucs_callbackq_add_oneshot(&ep->worker->uct->progress_q, ep, + ucp_wireup_tf_reply_progress, ep); +} + +static void ucp_wireup_tf_reply_finish(ucp_ep_h ep) +{ + ucp_ep_recovery_arg_t *arg = ucp_wireup_tf_recovery_arg(ep); + ucp_unpacked_address_t remote_address; + ucp_wireup_tf_reply_t *reply; + ucs_status_t status; + + reply = ucp_wireup_tf_reply_get(ep); + if ((reply == NULL) || (reply->waiting != 0)) { + return; + } + + /* Steal it before sending. A completion nested under the send must not + * finish this reply again. */ + arg->tf_reply = NULL; + memset(&remote_address, 0, sizeof(remote_address)); + if (reply->address != NULL) { + status = ucp_address_unpack(ep->worker, reply->address, + UCP_ADDRESS_PACK_FLAGS_ALL, + &remote_address); + if (status != UCS_OK) { + ucs_error("ep %p: deferred address unpack failed: %s", ep, + ucs_status_string(status)); + memset(&remote_address, 0, sizeof(remote_address)); + } + } + + ucp_wireup_lanes_addr_request_reply(ep, reply, &remote_address); + ucs_free(remote_address.address_list); + ucp_wireup_tf_reply_free(reply); +} + +static unsigned ucp_wireup_tf_reply_progress(void *arg) +{ + ucp_ep_h ep = arg; + ucp_wireup_tf_reply_t *reply = ucp_wireup_tf_reply_get(ep); + + if (reply == NULL) { + return 0; + } + + reply->scheduled = 0; + if (reply->waiting != 0) { + return 0; + } + + ucp_wireup_tf_reply_finish(ep); + return 1; +} + +static ucs_status_t +ucp_wireup_tf_reply_store(ucp_ep_h ep, const ucp_wireup_msg_lanes_info_t *info, + uint32_t request_id, ucp_lane_map_t skip_lanes, + ucp_lane_map_t waiting, + ucp_wireup_lane_token_t *rx_tokens, + const void *address, size_t address_length) +{ + ucp_ep_recovery_arg_t *arg = ucp_wireup_tf_recovery_arg(ep); + ucp_wireup_tf_reply_t *reply; + void *copy = NULL; + + if (arg == NULL) { + return UCS_ERR_UNSUPPORTED; + } + + if (address_length > 0) { + copy = ucs_malloc(address_length, "ucp_tf_reply_addr"); + if (copy == NULL) { + return UCS_ERR_NO_MEMORY; + } + + memcpy(copy, address, address_length); + } + + reply = arg->tf_reply; + if (reply == NULL) { + reply = ucs_calloc(1, sizeof(*reply), "ucp_tf_reply"); + if (reply == NULL) { + ucs_free(copy); + return UCS_ERR_NO_MEMORY; + } + + arg->tf_reply = reply; + } else { + ucp_wireup_free_lane_tokens(reply->provided_lane_map, reply->rx_tokens); + ucs_free(reply->address); + } + + reply->request_id = request_id; + reply->requested_lane_map = info->requested_lane_map; + reply->provided_lane_map = info->provided_lane_map; + reply->skip_lanes = skip_lanes; + reply->waiting |= waiting; + reply->address = copy; + reply->address_length = address_length; + memcpy(reply->rx_tokens, rx_tokens, sizeof(reply->rx_tokens)); + memset(rx_tokens, 0, sizeof(reply->rx_tokens)); + ucs_debug("ep %p: LANES_ADDR_REQ defer reply id 0x%x waiting 0x%" PRIx64, + ep, request_id, (uint64_t)reply->waiting); + return UCS_OK; +} + +static void ucp_wireup_tf_reply_disarm(ucp_ep_h ep, ucp_lane_map_t not_armed) +{ + ucp_wireup_tf_reply_t *reply = ucp_wireup_tf_reply_get(ep); + + if (reply == NULL) { + return; + } + + reply->waiting &= ~not_armed; + ucp_wireup_tf_reply_schedule(ep); +} + +void ucp_wireup_tf_lane_errored(ucp_ep_h ep, ucp_lane_index_t lane) +{ + ucp_wireup_tf_reply_t *reply = ucp_wireup_tf_reply_get(ep); + + if ((reply == NULL) || (lane >= UCP_MAX_LANES) || + !(reply->waiting & UCS_BIT(lane))) { + return; + } + + reply->waiting &= ~UCS_BIT(lane); + ucs_debug("ep %p: lane %d error, reply waiting 0x%" PRIx64, ep, lane, + (uint64_t)reply->waiting); + ucp_wireup_tf_reply_schedule(ep); +} + +void ucp_wireup_tf_reply_discard(ucp_ep_h ep) +{ + ucp_ep_recovery_arg_t *arg = ucp_wireup_tf_recovery_arg(ep); + ucp_wireup_tf_reply_t *reply; + + if ((arg == NULL) || (arg->tf_reply == NULL)) { + return; + } + + reply = arg->tf_reply; + if (reply->scheduled) { + ucs_callbackq_remove_oneshot(&ep->worker->uct->progress_q, ep, + ucp_wireup_tf_reply_remove_filter, ep); + } + + ucp_wireup_tf_reply_free(reply); + arg->tf_reply = NULL; +} + +static UCS_F_NOINLINE void ucp_wireup_process_lanes_addr_request( ucp_worker_h worker, ucp_ep_h ep, const ucp_wireup_msg_t *msg, const ucp_wireup_msg_lanes_info_t *lanes_info, uint32_t request_id, const ucp_wireup_lane_token_t *tx_tokens, - const ucp_unpacked_address_t *remote_address) + const ucp_unpacked_address_t *remote_address, const void *address, + size_t address_length) { - ucp_lane_map_t discard = 0; - ucp_lane_map_t peer_unaware, to_rebuild, peer_provided, invalidated; - ucp_lane_map_t pending, armed; + ucp_wireup_tf_reply_t immediate; ucp_wireup_lane_token_t rx_tokens[UCP_MAX_LANES]; + ucp_wireup_tf_reply_t *parked; + ucp_lane_map_t invalidated, pending, already, armed, stuck; ucs_status_t status; ucp_ep_update_remote_id(ep, msg->src_ep_id); /* A peer token means that lane already failed remotely. Derive the RX - * token while the transport endpoint is still installed, then move it - * to error. The completion holds it. The reply is sent now; the ACK - * purges it. */ + * token while the queue pair is still installed, then move it to error. + * Rebuild and the reply wait for that completion, which holds the lane. + * A request id behind one already accepted for that lane is ignored, so + * it cannot invalidate the replacement queue pair. */ memset(rx_tokens, 0, sizeof(rx_tokens)); invalidated = 0; + pending = 0; if (request_id != 0) { + if (!ucp_ep_tf_accept_peer_req(ep, lanes_info->provided_lane_map, + request_id)) { + ucs_debug("ep %p: LANES_ADDR_REQ id 0x%x is stale", ep, request_id); + return; + } + invalidated = ucp_ep_tf_invalidate_lanes(ep, lanes_info->provided_lane_map, 0, request_id); ucp_wireup_derive_rx_tokens(ep, lanes_info->provided_lane_map, - tx_tokens, rx_tokens); - pending = ucp_ep_tf_lanes_to_invalidate(ep, invalidated); - armed = ucp_ep_tf_invalidate_uct_eps(ep, pending); - discard = pending & ~armed; - } + tx_tokens, rx_tokens, 1); + /* The initiator timed out and sent a newer request. Drop the parked + * reply. Lanes still waiting were already moved to error and will not + * complete, so fail them and let this request rebuild. */ + parked = ucp_wireup_tf_reply_get(ep); + if ((parked != NULL) && (parked->waiting != 0) && + ucp_ep_tf_peer_req_is_newer((uint16_t)parked->request_id, + (uint16_t)request_id)) { + stuck = parked->waiting; + ucs_debug("ep %p: LANES_ADDR_REQ id 0x%x restarts reply id 0x%x" + " waiting 0x%" PRIx64, + ep, request_id, parked->request_id, (uint64_t)stuck); + ucp_wireup_tf_reply_discard(ep); + stuck = ucp_ep_tf_drop_invalidated(ep, stuck); + if (stuck != 0) { + ucp_ep_set_lanes_failed(ep, stuck, UCS_ERR_CONNECTION_RESET); + } + } - /* Lanes still up take the software path. A held lane is already failed - * and waits for the ACK. */ - peer_unaware = (lanes_info->provided_lane_map & - ~ucp_ep_get_failed_lanes(ep) & ~invalidated) | - discard; - if (peer_unaware != 0) { - ucs_debug("ep %p: LANES_ADDR_REQ triggering local failover for " - "asymmetric failure on lanes 0x%" PRIx64, - ep, (uint64_t)peer_unaware); - status = ucp_ep_failover_reconfig(ep, peer_unaware, - UCS_ERR_CONNECTION_RESET); - if (status != UCS_OK) { - ucs_diag("ep %p: local failover for LANES_ADDR_REQ failed: %s", - ep, ucs_status_string(status)); + already = 0; + if (ucp_wireup_tf_reply_get(ep) != NULL) { + already = ucp_wireup_tf_reply_get(ep)->waiting; } - } - to_rebuild = lanes_info->provided_lane_map & ucp_ep_get_failed_lanes(ep); - peer_provided = ucp_ep_recovery_rebuild_lanes(ep, to_rebuild, - remote_address); + pending = ucp_ep_tf_lanes_to_invalidate(ep, invalidated) & ~already; + } - ucs_debug("ep %p: LANES_ADDR_REQ requested=0x%" PRIx64 - " provided=0x%" PRIx64 " to_rebuild=0x%" PRIx64 - " rebuilt=0x%" PRIx64, - ep, lanes_info->requested_lane_map, lanes_info->provided_lane_map, - to_rebuild, peer_provided); + if ((pending != 0) || (ucp_wireup_tf_reply_get(ep) != NULL)) { + status = ucp_wireup_tf_reply_store(ep, lanes_info, request_id, + invalidated, pending, rx_tokens, + address, address_length); + if (status == UCS_OK) { + /* Installed before the invalidate, so a completion that arrives + * from it can finish the reply. */ + armed = ucp_ep_tf_invalidate_uct_eps(ep, pending); + ucp_wireup_tf_reply_disarm(ep, pending & ~armed); + ucp_wireup_free_lane_tokens(lanes_info->provided_lane_map, + rx_tokens); + return; + } - if (peer_provided != 0) { - ucp_wireup_eps_progress_sched(ep); + ucs_error("ep %p: failed to defer LANES_ADDR reply: %s", ep, + ucs_status_string(status)); + if (ucp_wireup_tf_reply_get(ep) != NULL) { + ucp_wireup_free_lane_tokens(lanes_info->provided_lane_map, + rx_tokens); + return; + } } - /* Always reply, even if peer_provided == 0, so the initiator knows the - * peer handled the request (empty provided_lane_map means "I couldn't - * satisfy any of the lanes you asked about"). The reply answers the lanes - * the request provided, so its RX tokens belong to them. */ - ucp_wireup_send_lanes_addr_msg(ep, UCP_WIREUP_MSG_LANES_ADDR_REPLY, - lanes_info->provided_lane_map, peer_provided, - request_id, rx_tokens); + memset(&immediate, 0, sizeof(immediate)); + immediate.request_id = request_id; + immediate.requested_lane_map = lanes_info->requested_lane_map; + immediate.provided_lane_map = lanes_info->provided_lane_map; + immediate.skip_lanes = invalidated; + memcpy(immediate.rx_tokens, rx_tokens, sizeof(immediate.rx_tokens)); + ucp_wireup_lanes_addr_request_reply(ep, &immediate, remote_address); ucp_wireup_free_lane_tokens(lanes_info->provided_lane_map, rx_tokens); } @@ -1467,7 +1783,7 @@ ucp_wireup_process_lanes_addr_reply( const ucp_unpacked_address_t *remote_address) { ucp_wireup_lane_token_t rx_tokens[UCP_MAX_LANES]; - ucp_lane_map_t rebuilt; + ucp_lane_map_t rebuilt, acked; ucp_ep_update_remote_id(ep, msg->src_ep_id); @@ -1496,18 +1812,20 @@ ucp_wireup_process_lanes_addr_reply( ucs_assert(request_id != 0); - /* Derive RX tokens from the reply TX tokens. They go back on the ACK and - * are not the tokens stored for our own purge. */ + /* Derive RX tokens from the reply TX tokens, including a lane whose + * exchange is already closed. They go back on the ACK. A destroyed + * queue pair leaves a hole. */ memset(rx_tokens, 0, sizeof(rx_tokens)); ucp_wireup_derive_rx_tokens(ep, lanes_info->provided_lane_map, tx_tokens, - rx_tokens); + rx_tokens, 0); + acked = ucp_wireup_lanes_with_tokens(lanes_info->provided_lane_map, + rx_tokens); ucp_wireup_send_lanes_addr_msg(ep, UCP_WIREUP_MSG_LANES_ADDR_ACK, lanes_info->provided_lane_map, 0, request_id, rx_tokens); ucp_wireup_free_lane_tokens(lanes_info->provided_lane_map, rx_tokens); ucp_ep_tf_lanes_purge_outstanding(ep, lanes_info->requested_lane_map, - request_id, 0, - lanes_info->requested_lane_map); + request_id, 0, acked); } /* Locate the lanes info and the optional token trailer of a LANES_ADDR @@ -1664,9 +1982,11 @@ static ucs_status_t ucp_wireup_msg_handler(void *arg, void *data, ucp_ep_set_lanes_failed_schedule(ep, 0, UCS_ERR_CONNECTION_RESET); } else if (msg->type == UCP_WIREUP_MSG_LANES_ADDR_REQUEST) { ucs_assert(lanes_info != NULL); - ucp_wireup_process_lanes_addr_request(worker, ep, msg, lanes_info, - request_id, tx_tokens, - &remote_address); + ucp_wireup_process_lanes_addr_request( + worker, ep, msg, lanes_info, request_id, tx_tokens, + &remote_address, address_ptr, + UCS_PTR_BYTE_DIFF(address_ptr, + UCS_PTR_BYTE_OFFSET(data, length))); } else if (msg->type == UCP_WIREUP_MSG_LANES_ADDR_REPLY) { ucs_assert(lanes_info != NULL); ucp_wireup_invalidate_unaware_failed_lanes(ep, lanes_info, @@ -1678,8 +1998,9 @@ static ucs_status_t ucp_wireup_msg_handler(void *arg, void *data, } else if (msg->type == UCP_WIREUP_MSG_LANES_ADDR_ACK) { ucs_assert(lanes_info != NULL); ucp_wireup_store_rx_tokens(ep, lanes_info, request_id, 1, rx_tokens); - ucs_debug("ep %p: LANES_ADDR_ACK request_id=0x%" PRIx32, ep, - request_id); + ucs_debug("ep %p: LANES_ADDR_ACK request_id=0x%" PRIx32 + " requested=0x%" PRIx64, + ep, request_id, lanes_info->requested_lane_map); ucp_ep_tf_lanes_purge_outstanding(ep, lanes_info->requested_lane_map, request_id, 1, lanes_info->requested_lane_map); diff --git a/src/ucp/wireup/wireup.h b/src/ucp/wireup/wireup.h index f9826cd74e6..1838f89e4e3 100644 --- a/src/ucp/wireup/wireup.h +++ b/src/ucp/wireup/wireup.h @@ -313,6 +313,15 @@ void ucp_wireup_send_lanes_addr_msg( const ucp_wireup_lane_token_t *rx_tokens); +/* An error completion arrived for @a lane. A LANES_ADDR reply that was + * waiting for this completion is sent once every such lane has completed. */ +void ucp_wireup_tf_lane_errored(ucp_ep_h ep, ucp_lane_index_t lane); + + +/* Drop a LANES_ADDR reply that is still waiting for error completions. */ +void ucp_wireup_tf_reply_discard(ucp_ep_h ep); + + /* Parse one token section into per-lane slots. @a slots may be NULL to only * measure the section, which is what @ref ucp_wireup_skip_token_section does. * gtest feeds both helpers crafted buffers. */ diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c index 41fc9da9f15..a07293ac10a 100644 --- a/src/ucp/wireup/wireup_cm.c +++ b/src/ucp/wireup/wireup_cm.c @@ -863,8 +863,8 @@ static void ucp_ep_cm_remote_disconnect_progress(ucp_ep_h ucp_ep) ucs_assert(ucp_ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED); if ((ucp_ep->flags & UCP_EP_FLAG_CLOSED) && - (ucp_ep->ext->close_req != NULL)) { - ucp_request_complete_send(ucp_ep->ext->close_req, UCS_OK); + (ucp_ep_close_req(ucp_ep) != NULL)) { + ucp_request_complete_send(ucp_ep_close_req(ucp_ep), UCS_OK); return; } @@ -908,17 +908,17 @@ static unsigned ucp_ep_cm_disconnect_progress(void *arg) - if close req is valid this is ucp_ep_close_nb request and it will be completed as the ep is destroyed, i.e. flushed and disconnected with any status */ - if (ucp_ep->ext->close_req != NULL) { + if (ucp_ep_close_req(ucp_ep) != NULL) { ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); } } else if (ucp_ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED) { ucp_ep_cm_remote_disconnect_progress(ucp_ep); - } else if (ucp_ep->ext->close_req != NULL) { + } else if (ucp_ep_close_req(ucp_ep) != NULL) { /* if the EP is not local connected, the EP has been closed and flushed, CM lane is disconnected, complete close request and destroy EP */ ucs_assert(ucp_ep->flags & UCP_EP_FLAG_CLOSED); ucp_ep_update_flags(ucp_ep, 0, UCP_EP_FLAG_REMOTE_CONNECTED); - close_req = ucp_ep->ext->close_req; + close_req = ucp_ep_close_req(ucp_ep); ucp_ep_local_disconnect_progress(close_req); /* don't touch UCP EP after local disconnect, since it is not valid * anymore */ diff --git a/test/gtest/ucp/test_ucp_fault_tolerance.cc b/test/gtest/ucp/test_ucp_fault_tolerance.cc index f9fd6c4d775..16672793301 100644 --- a/test/gtest/ucp/test_ucp_fault_tolerance.cc +++ b/test/gtest/ucp/test_ucp_fault_tolerance.cc @@ -201,6 +201,32 @@ class test_ucp_fault_tolerance : public test_ucp_memheap { } } + /* Replay the rc_x/PUT order 4, 0, 2, 1, 3, then 5. The last lane stays + * healthy. */ + void order_put_lanes(std::vector &lanes, + const std::string &lane_type) + { + static const ucp_lane_index_t order[] = {4, 0, 2, 1, 3, 5}; + + if (lanes.size() != (sizeof(order) / sizeof(order[0]))) { + UCS_TEST_SKIP_R(lane_type + "lane count is " + + std::to_string(lanes.size())); + } + + for (ucp_lane_index_t lane : order) { + if (std::find(lanes.begin(), lanes.end(), lane) == lanes.end()) { + UCS_TEST_SKIP_R(lane_type + "lane " + std::to_string(lane) + + " is missing"); + } + } + + lanes.assign(std::begin(order), std::end(order)); + for (ucp_lane_index_t lane : lanes) { + UCS_TEST_MESSAGE << lane_type << ": " << size_t(lane) << "/" + << lanes.size(); + } + } + ucp_ep_h get_ucp_ep_for_err_injection(failure_side_t failure_side) { return (failure_side == FAILURE_SIDE_INITIATOR) ? sender().ep(0, INJECTED_EP_INDEX) : receiver().ep(0, INJECTED_EP_INDEX); @@ -254,7 +280,13 @@ class test_ucp_fault_tolerance : public test_ucp_memheap { } std::vector lanes(tmp_lanes.begin(), tmp_lanes.end()); - shuffle_lanes(lanes, lane_type_str); + if ((op_mask & TEST_OP_PUT) && + !(op_mask & (TEST_OP_GET | TEST_OP_AM))) { + order_put_lanes(lanes, lane_type_str); + } else { + shuffle_lanes(lanes, lane_type_str); + } + check_proto_coverage(op_mask); return lanes; } @@ -881,7 +913,7 @@ class test_ucp_fault_tolerance : public test_ucp_memheap { while (ucs_get_time() < deadline) { short_progress_loop(); - arg = ep->ext->recovery_arg; + arg = ucp_ep_recovery_arg(ep); if (arg == NULL) { continue; } @@ -925,12 +957,11 @@ uct_completion_t *test_ucp_fault_tolerance::m_held_probe_comp = NULL; ucs::mock *test_ucp_fault_tolerance::m_probe_mock = NULL; unsigned test_ucp_fault_tolerance::m_probe_count = 0; -/* Same cases as test_ucp_fault_tolerance with UCX_FAILOVER_MODE=token. - * Disabled until token failover is complete. Enable with - * --gtest_also_run_disabled_tests. */ -class DISABLED_test_ucp_fault_tolerance_tf : public test_ucp_fault_tolerance { +/* Same cases as test_ucp_fault_tolerance with UCX_FAILOVER_MODE=token, but + * only on transports that support token based retransmits. */ +class test_ucp_fault_tolerance_tf : public test_ucp_fault_tolerance { public: - DISABLED_test_ucp_fault_tolerance_tf() { + test_ucp_fault_tolerance_tf() { modify_config("FAILOVER_MODE", "token"); /* IB MD key; set UCX_IB_INDIRECT_ATOMIC so token mode stays selected. */ modify_config("IB_INDIRECT_ATOMIC", "n", SETENV_IF_NOT_EXIST); @@ -1265,13 +1296,10 @@ void test_ucp_fault_tolerance::run_recovery_retries_exhausted_live_lanes() UCS_TEST_SKIP_R("no RC p2p lane was marked failed"); } - ASSERT_NE(nullptr, ep->ext->recovery_arg); - wait_for_cond([ep]() { - return ep->ext->recovery_arg == NULL; - }, [this]() { - short_progress_loop(); - }); - ASSERT_EQ(nullptr, ep->ext->recovery_arg) + ASSERT_NE(nullptr, ucp_ep_recovery_arg(ep)); + wait_for_cond([ep]() { return ucp_ep_recovery_arg(ep) == NULL; }, + [this]() { short_progress_loop(); }); + ASSERT_EQ(nullptr, ucp_ep_recovery_arg(ep)) << "recovery retries were not exhausted"; EXPECT_EQ(0, total_err_count()) @@ -1305,10 +1333,10 @@ void test_ucp_fault_tolerance::run_recovery_retries_exhausted_live_lanes() "KEEPALIVE_INTERVAL=0.1s") UCP_FT_TESTS(test_ucp_fault_tolerance) -UCP_FT_TESTS(DISABLED_test_ucp_fault_tolerance_tf) +UCP_FT_TESTS(test_ucp_fault_tolerance_tf) #undef UCP_FT_TEST #undef UCP_FT_TESTS UCP_INSTANTIATE_TEST_CASE(test_ucp_fault_tolerance) -UCP_INSTANTIATE_TEST_CASE(DISABLED_test_ucp_fault_tolerance_tf) +UCP_INSTANTIATE_TEST_CASE_TLS(test_ucp_fault_tolerance_tf, rcx, "rc_x")