UCP/WIREUP: Add a token trailer and an ACK message to the lane address exchange

src/ucp/core/ucp_ep.c

@@ -2274,8 +2274,15 @@ static int ucp_ep_recovery_send_request(ucp_ep_h ep)
     ucs_debug("ep %p: sending recovery request, failed=0x%" PRIx64
               " recovery=0x%" PRIx64, ep, failed_lanes, recovery_lanes);
 
-    ucp_wireup_send_lanes_addr_msg(ep, UCP_WIREUP_MSG_LANES_ADDR_REQUEST,
-                                   failed_lanes, recovery_lanes);
+    /* A request answers no other message, so requested_lane_map is empty. */
+    if (++ep->ext->recovery_arg->request_id == 0) {
+        /* 0 is the no-trailer sentinel on the wire */
+        ep->ext->recovery_arg->request_id = 1;
+    }
+
+    ucp_wireup_send_lanes_addr_msg(ep, UCP_WIREUP_MSG_LANES_ADDR_REQUEST, 0,
+                                   recovery_lanes,
+                                   ep->ext->recovery_arg->request_id);
     return 1;
 }
 

src/ucp/core/ucp_ep.h

@@ -513,6 +513,11 @@ typedef struct ucp_ep_recovery_arg {
     /* number of retries left before giving up */
     unsigned                retries_left;
     uint8_t                 state;
+    /* Generation of the LANES_ADDR exchange, pre-incremented by every request
+     * and echoed by the peer in its answers. Only carried on the wire for now,
+     * the follow-up patch matches it against the tokens of an answer to tell
+     * apart the round they belong to */
+    uint32_t                request_id;
     ucp_ep_recovery_probe_t probe[UCP_MAX_LANES];
 } ucp_ep_recovery_arg_t;
 

src/ucp/wireup/AGENTS.md

@@ -9,7 +9,10 @@ exchanged out-of-band) and connection-manager wireup (sockaddr listener).
 
 - `wireup.[ch]` — the wireup state machine. Defines the on-wire message
   types `UCP_WIREUP_MSG_{PRE_REQUEST, REQUEST, REPLY, ACK, EP_CHECK,
-  EP_REMOVED, REPLY_RECONFIG}` and drives transitions on send/recv.
+  EP_REMOVED, REPLY_RECONFIG, LANES_ADDR_REQUEST, LANES_ADDR_REPLY,
+  LANES_ADDR_ACK}` and drives transitions on send/recv. The `LANES_ADDR`
+  family re-exchanges the addresses of failed lanes during recovery, and
+  carries the trailer of the UCT lane tokens used to purge them.
 - `address.[ch]` — packing and unpacking of `ucp_address_t`: per-device
   records, per-iface records, atomic capability bits, system-device IDs,
   and connection info. ABI-sensitive: any layout change must add a version
@@ -41,6 +44,15 @@ exchanged out-of-band) and connection-manager wireup (sockaddr listener).
   through reserved bits/length fields, never field reordering. See the
   `UCP_WIREUP_MSG_*` constants in `wireup.h` and the matching pack/unpack
   in `wireup.c`.
+- A payload section appended to an already released message must be gated
+  by a `*_MIN_DST_VERSION` constant checked against
+  `ucp_ep_config(ep)->key.dst_version`. Variable-length pieces go before
+  the rest of the payload so an older peer, which never sees them, still
+  finds the original contents. `LANES_ADDR` token trailers work this way,
+  see `UCP_WIREUP_ADDR_TOKEN_MIN_DST_VERSION`: each token is prefixed by
+  its own length, and the addresses stay last. Trailer lanes are taken
+  from `ucp_wireup_msg_lanes_info_t` rather than from maps of their own,
+  so keep the two in sync when changing either.
 - New transport capabilities advertised via address records require both
   a packer in `address.c` and a consumer in `select.c`. Forgetting one
   half silently disables the capability for the new build vs. old peers.