issue: 4219010 XLIO support for kernel 6.10

Kernel 6.10 netlink has breaked XLIO functionality.
Transitioned to libnl - an abstraction that wraps netlink.

This both solves the issue and makes us more robust.

Signed-off-by: Tomer Cabouly <[email protected]>

Author: tomerdbz

src/core/proto/netlink_socket_mgr.cpp

@@ -32,173 +32,72 @@
  * SOFTWARE.
  */
 
-#include "core/sock/sock-redirect.h"
 #include "core/util/utils.h"
 #include "vlogger/vlogger.h"
 #include "utils/bullseye.h"
 #include "netlink_socket_mgr.h"
 
 #include <linux/rtnetlink.h>
 #include <linux/netlink.h>
-#include <stdlib.h>
 #include <unistd.h> // getpid()
 
+#include <netlink/route/route.h>
+#include <netlink/route/rule.h>
+#include <netlink/route/link.h>
+
 #ifndef MODULE_NAME
 #define MODULE_NAME "netlink_socket_mgr:"
 #endif
 
-#define MSG_BUFF_SIZE 81920
-
-// This function builds Netlink request to retrieve data (Rule, Route) from kernel.
-// Parameters :
-//      data_type   : either RULE_DATA_TYPE or ROUTE_DATA_TYPE
-//      pid         : opaque pid for netlink request
-//      seq         : opaque seq for netlink request
-//      buf         : buffer for the request
-//      nl_msg      : [out] pointer to request
-void netlink_socket_mgr::build_request(nl_data_t data_type, uint32_t pid, uint32_t seq, char *buf,
-                                       struct nlmsghdr **nl_msg)
-{
-    struct rtmsg *rt_msg;
-
-    assert(MSG_BUFF_SIZE >= NLMSG_SPACE(sizeof(struct rtmsg)));
-    memset(buf, 0, NLMSG_SPACE(sizeof(struct rtmsg)));
-
-    // point the header and the msg structure pointers into the buffer
-    *nl_msg = (struct nlmsghdr *)buf;
-    rt_msg = (struct rtmsg *)NLMSG_DATA(*nl_msg);
-
-    // Fill in the nlmsg header
-    (*nl_msg)->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
-    (*nl_msg)->nlmsg_seq = seq;
-    (*nl_msg)->nlmsg_pid = pid;
-    (*nl_msg)->nlmsg_type = data_type == RULE_DATA_TYPE ? RTM_GETRULE : RTM_GETROUTE;
-    (*nl_msg)->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST;
-
-    rt_msg->rtm_family = AF_UNSPEC;
-}
-
-// Query built request and receive requested data (Rule, Route)
-// Parameters:
-//      nl_msg  : request that is built previously.
-//      buf     : [out] buffer for the reply
-//      len     : [out] length of received data.
-bool netlink_socket_mgr::query(const struct nlmsghdr *nl_msg, char *buf, int &len)
+// Update data in a table
+void netlink_socket_mgr::update_tbl(nl_data_t data_type)
 {
-    int sockfd;
-
-    // Opaque information in the request. To track expected reply.
-    uint32_t nl_pid = nl_msg->nlmsg_pid;
-    uint32_t nl_seq = nl_msg->nlmsg_seq;
+    nl_sock *sockfd = nullptr;
 
     BULLSEYE_EXCLUDE_BLOCK_START
-    if ((sockfd = SYSCALL(socket, PF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE)) < 0) {
-        __log_err("NL socket creation failed, errno = %d", errno);
-        return false;
-    }
-    if (SYSCALL(fcntl, sockfd, F_SETFD, FD_CLOEXEC) != 0) {
-        __log_warn("Fail in fcntl, errno = %d", errno);
-    }
-    if ((len = SYSCALL(send, sockfd, nl_msg, nl_msg->nlmsg_len, 0)) < 0) {
-        __log_err("Write to NL socket failed, errno = %d", errno);
-    }
-    if (len > 0 && (len = recv_info(sockfd, nl_pid, nl_seq, buf)) < 0) {
-        __log_err("Read from NL socket failed...");
-    }
-    BULLSEYE_EXCLUDE_BLOCK_END
 
-    close(sockfd);
-    return len > 0;
-}
+    sockfd = nl_socket_alloc();
+    if (sockfd == nullptr) {
+        __log_err("NL socket Creation: ");
+        throw_xlio_exception("Failed nl_socket_alloc");
+    }
 
-// Receive requested data and save it to buffer.
-// Return length of received data.
-// Parameters:
-//      sockfd  : netlink socket
-//      pid     : expected opaque pid value
-//      seq     : expected opaque seq value
-//      buf     : [out] read reply
-int netlink_socket_mgr::recv_info(int sockfd, uint32_t pid, uint32_t seq, char *buf)
-{
-    struct nlmsghdr *nlHdr;
-    int readLen;
-    int msgLen = 0;
-    char *buf_ptr = buf;
-
-    do {
-        // Receive response from the kernel
-        BULLSEYE_EXCLUDE_BLOCK_START
-        if ((readLen = SYSCALL(recv, sockfd, buf_ptr, MSG_BUFF_SIZE - msgLen, 0)) < 0) {
-            __log_err("NL socket read failed, errno = %d", errno);
-            return -1;
-        }
-
-        nlHdr = (struct nlmsghdr *)buf_ptr;
-
-        // Check if the header is valid
-        if ((NLMSG_OK(nlHdr, (u_int)readLen) == 0) || (nlHdr->nlmsg_type == NLMSG_ERROR)) {
-            __log_err("Error in received packet, readLen = %d, msgLen = %d, type=%d, bufLen = %d",
-                      readLen, nlHdr->nlmsg_len, nlHdr->nlmsg_type, MSG_BUFF_SIZE);
-            if ((int)nlHdr->nlmsg_len >= MSG_BUFF_SIZE - msgLen) {
-                __log_err("The buffer we pass to netlink is too small for reading the whole table");
-            }
-            return -1;
-        }
-        BULLSEYE_EXCLUDE_BLOCK_END
-
-        if ((nlHdr->nlmsg_seq != seq) || (nlHdr->nlmsg_pid != pid)) {
-            // Skip not expected messages
-            continue;
-        }
-
-        buf_ptr += readLen;
-        msgLen += readLen;
-
-        // Loop until this is the last message of expected reply
-    } while (nlHdr->nlmsg_type != NLMSG_DONE && (nlHdr->nlmsg_flags & NLM_F_MULTI));
-
-    return msgLen;
-}
+    if (nl_connect(sockfd, NETLINK_ROUTE) < 0) {
+        __log_err("NL socket Connection: ");
+        nl_socket_free(sockfd);
+        throw_xlio_exception("Failed nl_connect");
+    }
 
-// Update data in a table
-void netlink_socket_mgr::update_tbl(nl_data_t data_type)
-{
-    struct nlmsghdr *nl_msg = nullptr;
-    char *buf;
-    int len = 0;
+    struct nl_cache *cache_state = {0};
+    int err = 0;
 
-    // Opaque netlink information
-    uint32_t nl_pid = getpid();
-    uint32_t nl_seq = static_cast<uint32_t>(data_type);
+    // cache allocation fetches the latest existing rules/routes
+    if (data_type == RULE_DATA_TYPE) {
 
-    __log_dbg("");
+        err = rtnl_rule_alloc_cache(sockfd, AF_INET, &cache_state);
+    } else if (data_type == ROUTE_DATA_TYPE) {
 
-    buf = new char[MSG_BUFF_SIZE];
-    if (!buf) {
-        __log_err("NL message buffer allocation failed");
-        return;
+        err = rtnl_route_alloc_cache(sockfd, AF_INET, 0, &cache_state);
     }
 
-    build_request(data_type, nl_pid, nl_seq, buf, &nl_msg);
-    if (query(nl_msg, buf, len)) {
-        parse_tbl(buf, len);
+    if (err < 0) {
+        throw_xlio_exception("Failed to allocate route cache");
     }
 
-    delete[] buf;
-    __log_dbg("Done");
+    parse_tbl(cache_state);
 }
 
 // Parse received data in a table
-// Parameters:
-//      buf : buffer with netlink reply.
-//      len : length of received data.
-void netlink_socket_mgr::parse_tbl(char *buf, int len)
+void netlink_socket_mgr::parse_tbl(struct nl_cache *cache_state)
 {
-    struct nlmsghdr *nl_header = (struct nlmsghdr *)buf;
-
-    for (; NLMSG_OK(nl_header, (u_int)len); nl_header = NLMSG_NEXT(nl_header, len)) {
-        parse_entry(nl_header);
-    }
+    // a lambda can't be casted to a c-fptr with ref captures - so we provide context ourselves
+    nl_cache_foreach(
+        cache_state,
+        [](struct nl_object *nl_obj, void *context) {
+            netlink_socket_mgr *this_ptr = reinterpret_cast<netlink_socket_mgr *>(context);
+            this_ptr->parse_entry(nl_obj);
+        },
+        this);
 }
 
 #undef MODULE_NAME

src/core/proto/netlink_socket_mgr.h

@@ -34,25 +34,18 @@
 
 #ifndef NETLINK_SOCKET_MGR_H
 #define NETLINK_SOCKET_MGR_H
-
-// Forward declarations
-struct nlmsghdr;
-
+#include <netlink/netlink.h>
 // This enum specify the type of data to be retrieve using netlink socket.
 enum nl_data_t { RULE_DATA_TYPE, ROUTE_DATA_TYPE };
 
 // This class manages retrieving data (Rule, Route) from kernel using netlink socket.
 class netlink_socket_mgr {
 protected:
-    virtual void parse_entry(struct nlmsghdr *nl_header) = 0;
+    virtual void parse_entry(struct nl_object *nl_obj) = 0;
     virtual void update_tbl(nl_data_t data_type);
 
 private:
-    void build_request(nl_data_t data_type, uint32_t pid, uint32_t seq, char *buf,
-                       struct nlmsghdr **nl_msg);
-    bool query(const struct nlmsghdr *nl_msg, char *buf, int &len);
-    int recv_info(int sockfd, uint32_t pid, uint32_t seq, char *buf);
-    void parse_tbl(char *buf, int len);
+    void parse_tbl(struct nl_cache *cache_state);
 };
 
 #endif /* NETLINK_SOCKET_MGR_H */