[TENT] Cover the RDMA context enable failure paths and link-speed events without an RNIC

mooncake-transfer-engine/tent/include/tent/transport/rdma/ibv_loader.h

@@ -29,6 +29,13 @@ struct IbvSymbols {
                             struct ibv_device_attr* device_attr);
     int (*ibv_query_gid)(struct ibv_context* context, uint8_t port_num,
                          int index, union ibv_gid* gid);
+    // The entry point behind the ibv_query_gid_ex() header inline. Taken
+    // through the table like the rest so GID scanning can be stood in for;
+    // call it through QueryGidEx() below rather than passing entry_size by
+    // hand.
+    int (*ibv_query_gid_ex)(struct ibv_context* context, uint32_t port_num,
+                            uint32_t gid_index, struct ibv_gid_entry* entry,
+                            uint32_t flags, size_t entry_size);
     int (*ibv_query_port_default)(ibv_context* context, uint8_t port_num,
                                   ibv_port_attr* port_attr);
     // Optional (rdma-core >= 62): effective port speed in 100 Mb/s units,
@@ -66,6 +73,15 @@ struct IbvSymbols {
     int (*ibv_fork_init)(void);
 };
 
+// ibv_query_gid_ex() as the header defines it, but dispatched through a
+// context's own symbol table.
+inline int QueryGidEx(const IbvSymbols& verbs, ibv_context* context,
+                      uint32_t port_num, uint32_t gid_index,
+                      ibv_gid_entry* entry) {
+    return verbs.ibv_query_gid_ex(context, port_num, gid_index, entry,
+                                  /*flags=*/0, sizeof(*entry));
+}
+
 class IbvLoader {
    public:
     static IbvLoader& Instance();

mooncake-transfer-engine/tent/include/tent/transport/rdma/workers.h

@@ -37,6 +37,19 @@ namespace tent {
 class RdmaTransport;
 class DeviceSelector;
 
+// IBV_EVENT_DEVICE_SPEED_CHANGE: a port speed changed without a link flap
+// (a VF over LAG losing a PF, say). Newer rdma-core names it; the async
+// event ABI is append-only, so an older header means only that the
+// enumerator is missing, never that the kernel cannot raise the event.
+// Naming it here keeps the branch that handles it -- and its test --
+// compiled everywhere, so an old userspace on a new kernel still reacts.
+inline constexpr int kIbvEventDeviceSpeedChange = 20;  // after WQ_FATAL (19)
+#ifdef HAVE_IBV_EVENT_DEVICE_SPEED_CHANGE
+// A build against a header that names it pins the value above.
+static_assert(kIbvEventDeviceSpeedChange == IBV_EVENT_DEVICE_SPEED_CHANGE,
+              "verbs async-event values are append-only; update the fallback");
+#endif
+
 class Workers {
     friend class RdmaTransportTestPeer;
 

mooncake-transfer-engine/tent/src/transport/rdma/context.cpp

@@ -198,6 +198,7 @@ static inline std::string gidBytesToString(const uint8_t* raw) {
 // 3) RoCEv2/IB + no network device
 // 4) first non-zero GID (any type)
 static inline GidNetworkState getBestGidIndex(const std::string& device_name,
+                                              const IbvSymbols& verbs,
                                               struct ibv_context* context,
                                               ibv_port_attr& port_attr,
                                               uint8_t port, int& gid_index) {
@@ -213,7 +214,7 @@ static inline GidNetworkState getBestGidIndex(const std::string& device_name,
             << device_name << " port " << (int)port;
 
     for (i = 0; i < port_attr.gid_tbl_len; i++) {
-        if (ibv_query_gid_ex(context, port, i, &gid_entry, 0)) {
+        if (QueryGidEx(verbs, context, port, i, &gid_entry)) {
             continue;  // Skip invalid GID entries
         }
 
@@ -699,7 +700,8 @@ RdmaAddressRefreshResult RdmaContext::refreshAddress(
 
     int next_gid_index = params_->device.gid_index;
     if (next_gid_index < 0) {
-        if (getBestGidIndex(device_name_, native_context_, port_attr, port,
+        if (getBestGidIndex(device_name_, verbs_, native_context_, port_attr,
+                            port,
                             next_gid_index) == GidNetworkState::GID_NOT_FOUND) {
             LOG(WARNING) << "No suitable GID found while refreshing "
                          << device_name_ << "/" << static_cast<int>(port);
@@ -839,7 +841,7 @@ int RdmaContext::openDevice(const std::string& device_name, uint8_t port) {
     if (params_->verbose) {
         for (int i = 0; i < port_attr.gid_tbl_len; i++) {
             struct ibv_gid_entry entry;
-            if (ibv_query_gid_ex(context.get(), port, i, &entry, 0)) {
+            if (QueryGidEx(verbs_, context.get(), port, i, &entry)) {
                 PLOG(WARNING)
                     << "Scan: Unable to query GID " << i << " on device "
                     << device_name << " port " << port;
@@ -858,8 +860,9 @@ int RdmaContext::openDevice(const std::string& device_name, uint8_t port) {
     if (gid_index_ < 0) {
         // Auto-select GID
         int found_gid_index = -1;
-        GidNetworkState gid_state = getBestGidIndex(
-            device_name, context.get(), port_attr, port, found_gid_index);
+        GidNetworkState gid_state =
+            getBestGidIndex(device_name, verbs_, context.get(), port_attr, port,
+                            found_gid_index);
         if (gid_state == GidNetworkState::GID_NOT_FOUND) {
             LOG(ERROR) << "No valid GID found for device " << device_name
                        << " port " << static_cast<int>(port);
@@ -894,8 +897,8 @@ int RdmaContext::openDevice(const std::string& device_name, uint8_t port) {
             has_issues = true;
         }
 
-        if (ibv_query_gid_ex(context.get(), port, gid_index_, &user_gid_entry,
-                             0) == 0) {
+        if (QueryGidEx(verbs_, context.get(), port, gid_index_,
+                       &user_gid_entry) == 0) {
             bool is_ipv4 =
                 ipv6_addr_v4mapped((struct in6_addr*)user_gid_entry.gid.raw);
             bool is_roce_v2 = user_gid_entry.gid_type == IBV_GID_TYPE_ROCE_V2;

mooncake-transfer-engine/tent/src/transport/rdma/ibv_loader.cpp

@@ -66,6 +66,9 @@ IbvLoader::IbvLoader() {
     ok &= LoadSymbol(handle_, "ibv_close_device", symbols_.ibv_close_device);
     ok &= LoadSymbol(handle_, "ibv_query_device", symbols_.ibv_query_device);
     ok &= LoadSymbol(handle_, "ibv_query_gid", symbols_.ibv_query_gid);
+    // ibv_query_gid_ex() is a header inline over this entry point, which the
+    // binary already links, so requiring it here loses no host.
+    ok &= LoadSymbol(handle_, "_ibv_query_gid_ex", symbols_.ibv_query_gid_ex);
     ok &=
         LoadSymbol(handle_, "ibv_query_port", symbols_.ibv_query_port_default);
     LoadOptionalSymbol(handle_, "ibv_query_port_speed",

mooncake-transfer-engine/tent/src/transport/rdma/workers.cpp

@@ -1230,7 +1230,10 @@ int Workers::handleContextEvents(int dev_id,
 
 void Workers::applyContextEvent(int dev_id, RdmaContext& context,
                                 const ibv_async_event& event) {
-    switch (event.event_type) {
+    // Switched as an int, not as the enum: kIbvEventDeviceSpeedChange is not
+    // an enumerator on older headers, and the default below already covers
+    // every event this does not act on.
+    switch (static_cast<int>(event.event_type)) {
         case IBV_EVENT_QP_FATAL:
         case IBV_EVENT_WQ_FATAL: {
             auto endpoint = (RdmaEndPoint*)event.element.qp->qp_context;
@@ -1287,15 +1290,14 @@ void Workers::applyContextEvent(int dev_id, RdmaContext& context,
             }
             break;
         }
-#ifdef HAVE_IBV_EVENT_DEVICE_SPEED_CHANGE
-        case IBV_EVENT_DEVICE_SPEED_CHANGE:
-            // rdma-core >= 62: a port speed changed without a link flap
-            // (e.g. a VF over LAG losing a PF). Device-level, so the event
-            // names no port; each context opens exactly one, so re-query
-            // that one.
+        case kIbvEventDeviceSpeedChange:
+            // A port speed changed without a link flap (e.g. a VF over LAG
+            // losing a PF). Device-level, so the event names no port; each
+            // context opens exactly one, so re-query that one. See
+            // kIbvEventDeviceSpeedChange for why this is not behind an
+            // #ifdef on the header's enumerator.
             refreshLinkSpeed(dev_id, context);
             break;
-#endif
         default:
             break;
     }