rework termination mac handling

pkg/systemd/sysconfig.template

@@ -9,8 +9,11 @@
 # gRPC listening port:
 # FLAGS_ofdpa_grpc_port=50051
 #
-# Use KNET interfaces (experimental):
-# FLAGS_use_knet=false
+# Use KNET interfaces:
+# FLAGS_use_knet=true
+#
+# Mark switched packets as offloaded:
+# FLAGS_mark_fwd_offload=true
 
 ### glog
 #

src/baseboxd.cc

@@ -19,7 +19,8 @@ DECLARE_string(tryfromenv); // from gflags
 DEFINE_bool(multicast, true, "Enable multicast support");
 DEFINE_int32(port, 6653, "Listening port");
 DEFINE_int32(ofdpa_grpc_port, 50051, "Listening port of ofdpa gRPC server");
-DEFINE_bool(use_knet, false, "Use KNET interfaces (experimental)");
+DEFINE_bool(use_knet, true, "Use KNET interfaces");
+DEFINE_bool(mark_fwd_offload, true, "Mark switched packets as offloaded");
 
 static bool validate_port(const char *flagname, gflags::int32 value) {
   VLOG(3) << __FUNCTION__ << ": flagname=" << flagname << ", value=" << value;
@@ -48,7 +49,8 @@ int main(int argc, char **argv) {
   }
 
   // all variables can be set from env
-  FLAGS_tryfromenv = std::string("multicast,port,ofdpa_grpc_port,use_knet");
+  FLAGS_tryfromenv =
+      std::string("multicast,port,ofdpa_grpc_port,use_knet,mark_fwd_offload");
   gflags::SetUsageMessage("");
   gflags::SetVersionString(PROJECT_VERSION);
 

src/netlink/cnetlink.cc

@@ -42,6 +42,7 @@
 #include "nl_vxlan.h"
 
 DECLARE_bool(multicast);
+DECLARE_bool(mark_fwd_offload);
 
 namespace basebox {
 
@@ -567,6 +568,8 @@ int cnetlink::add_l3_configuration(rtnl_link *link) {
 
   // add all ip addresses and routes from collected interfaces
   for (auto l : links) {
+    add_termination_mac(l);
+
     rv = add_l3_addresses(l);
     if (rv < 0)
       LOG(WARNING) << __FUNCTION__ << ": failed to add l3 addresses (" << rv
@@ -600,22 +603,53 @@ int cnetlink::remove_l3_configuration(rtnl_link *link) {
     if (rv < 0)
       LOG(WARNING) << __FUNCTION__ << ": failed to remove l3 addresses (" << rv
                    << " from link " << OBJ_CAST(l);
+    remove_termination_mac(l);
   }
 
   return rv;
 }
 
+int cnetlink::add_termination_mac(rtnl_link *link) {
+  struct nl_addr *addr = rtnl_link_get_addr(link);
+  auto mac = rofl::caddress_ll((uint8_t *)nl_addr_get_binary_addr(addr),
+                               nl_addr_get_len(addr));
+  uint32_t port_id = get_port_id(link);
+  uint16_t vid = 0;
+
+  if (rtnl_link_is_vlan(link))
+    vid = rtnl_link_vlan_get_id(link);
+
+  swi->l3_termination_add(port_id, vid, mac);
+  swi->l3_termination_add_v6(port_id, vid, mac);
+
+  return 0;
+}
+
+int cnetlink::remove_termination_mac(rtnl_link *link) {
+  struct nl_addr *addr = rtnl_link_get_addr(link);
+  auto mac = rofl::caddress_ll((uint8_t *)nl_addr_get_binary_addr(addr),
+                               nl_addr_get_len(addr));
+  uint32_t port_id = get_port_id(link);
+  uint16_t vid = 0;
+
+  if (rtnl_link_is_vlan(link))
+    vid = rtnl_link_vlan_get_id(link);
+
+  swi->l3_termination_remove_v6(port_id, vid, mac);
+  swi->l3_termination_remove(port_id, vid, mac);
+
+  return 0;
+}
+
 int cnetlink::update_on_mac_change(rtnl_link *old_link, rtnl_link *new_link) {
   int rv = 0;
   int port_id = get_port_id(old_link);
   uint16_t vid = vlan->get_vid(old_link);
   struct nl_addr *old_mac = rtnl_link_get_addr(old_link);
   struct nl_addr *new_mac = rtnl_link_get_addr(new_link);
 
-  rv = l3->update_l3_termination(port_id, vid, old_mac, new_mac);
-  if (rv < 0)
-    VLOG(1) << __FUNCTION__ << ": failed to update termination MAC, old link="
-            << OBJ_CAST(old_link) << " new link=" << OBJ_CAST(new_link);
+  remove_termination_mac(old_link);
+  add_termination_mac(new_link);
 
   // In response to the MAC address change on the interface, linux deletes the
   // neighbors configured on the interface. We are tracking the state
@@ -1342,6 +1376,8 @@ void cnetlink::link_created(rtnl_link *link) noexcept {
     VLOG(1) << __FUNCTION__ << ": new vlan interface " << OBJ_CAST(link);
     uint16_t vid = rtnl_link_vlan_get_id(link);
     vlan->add_vlan(link, vid, true);
+    if (is_switch_interface(link))
+      add_termination_mac(link);
   } break;
   case LT_BOND: {
     VLOG(1) << __FUNCTION__ << ": new bond interface " << OBJ_CAST(link);
@@ -1352,9 +1388,16 @@ void cnetlink::link_created(rtnl_link *link) noexcept {
   } break;
   default: {
     bool handled = port_man->portdev_ready(link);
-    if (!handled)
+    if (handled) {
+      uint32_t port_id = get_port_id(link);
+
+      port_man->set_offloaded(link, FLAGS_mark_fwd_offload);
+      swi->port_set_config(port_id, port_man->get_hwaddr(port_id), false);
+      add_termination_mac(link);
+    } else {
       LOG(WARNING) << __FUNCTION__ << ": ignoring link with lt=" << lt
                    << " link:" << OBJ_CAST(link);
+    }
   } break;
   } // switch link type
 }
@@ -1399,12 +1442,18 @@ void cnetlink::link_updated(rtnl_link *old_link, rtnl_link *new_link) noexcept {
     return;
   }
 
+  uint32_t port_id = port_man->get_port_id(rtnl_link_get_ifindex(new_link));
+  if (port_id > 0 && (rtnl_link_get_flags(old_link) & IFF_UP) !=
+                         (rtnl_link_get_flags(new_link) & IFF_UP)) {
+    swi->port_set_config(port_id, port_man->get_hwaddr(port_id),
+                         !!(rtnl_link_get_flags(new_link) & IFF_UP));
+  }
+
   switch (lt_old) {
   case LT_BOND_SLAVE:
     if (lt_new == LT_BOND_SLAVE) { // bond slave updated
       bond->update_lag_member(old_link, new_link);
-    } else if (port_man->get_port_id(rtnl_link_get_ifindex(new_link)) >
-               0) { // bond slave removed
+    } else if (port_id > 0) { // bond slave removed
       bond->remove_lag_member(old_link);
     }
     break;
@@ -1465,7 +1514,7 @@ void cnetlink::link_updated(rtnl_link *old_link, rtnl_link *new_link) noexcept {
             << ", new link: " << OBJ_CAST(new_link);
     break;
   default:
-    if (port_man->get_port_id(rtnl_link_get_ifindex(new_link)) > 0) {
+    if (port_id > 0) {
       if (lt_new == LT_BOND_SLAVE) {
         // XXX link enslaved
         LOG(INFO) << __FUNCTION__ << ": link enslaved "
@@ -1521,6 +1570,7 @@ void cnetlink::link_deleted(rtnl_link *link) noexcept {
       bridge->clear_tpid_entries(); // clear the Egress TPID table
       delete bridge;
       bridge = nullptr;
+      remove_termination_mac(link);
     }
     break;
   case LT_VXLAN: {
@@ -1537,6 +1587,8 @@ void cnetlink::link_deleted(rtnl_link *link) noexcept {
   case LT_VLAN:
     VLOG(1) << __FUNCTION__ << ": removed vlan interface " << OBJ_CAST(link);
     vlan->remove_vlan(link, rtnl_link_vlan_get_id(link), true);
+    if (is_switch_interface(link))
+      remove_termination_mac(link);
     break;
   case LT_BOND: {
     VLOG(1) << __FUNCTION__ << ": removed bond interface " << OBJ_CAST(link);

src/netlink/cnetlink.h

@@ -66,6 +66,9 @@ class cnetlink final : public rofl::cthread_env {
   int add_l3_configuration(rtnl_link *link);
   int remove_l3_configuration(rtnl_link *link);
 
+  int add_termination_mac(rtnl_link *link);
+  int remove_termination_mac(rtnl_link *link);
+
   int update_on_mac_change(rtnl_link *old_link, rtnl_link *new_link);
 
   bool has_l3_addresses(rtnl_link *link);

src/netlink/knet_manager.cc

@@ -84,8 +84,15 @@ int knet_manager::create_portdev(uint32_t port_id, const std::string &port_name,
         auto rv = port_names2id.emplace(std::make_pair(port_name, port_id));
 
         if (!rv.second) {
-          LOG(FATAL) << __FUNCTION__ << ": failed to insert";
+          LOG(FATAL) << __FUNCTION__ << ": failed to insert port name";
         }
+
+        auto rv2 = id_to_hwaddr.emplace(std::make_pair(port_id, hwaddr));
+
+        if (!rv2.second) {
+          LOG(FATAL) << __FUNCTION__ << ": failed to insert hwaddr";
+        }
+
         r = system(("/usr/sbin/client_drivshell knet netif create port=" +
                     std::to_string(port_id) + " ifname=" + port_name +
                     " mac=" + mac_string + " keeprxtag=yes")
@@ -313,4 +320,15 @@ int knet_manager::set_port_speed(const std::string name, uint32_t speed,
   return 1;
 }
 
+int knet_manager::set_offloaded(rtnl_link *link, bool offloaded) {
+  std::string name(rtnl_link_get_name(link));
+  std::ofstream file("/proc/bcm/knet/link");
+
+  if (file.is_open()) {
+    file << (name + (offloaded ? "=offload" : "=no-offload"));
+    file.close();
+  }
+  return 0;
+}
+
 } // namespace basebox

src/netlink/knet_manager.h

@@ -37,6 +37,7 @@ class knet_manager final : public port_manager {
 
   int change_port_status(const std::string name, bool status);
   int set_port_speed(const std::string name, uint32_t speed, uint8_t duplex);
+  int set_offloaded(rtnl_link *link, bool offloaded);
 
   // access from northbound (cnetlink)
   bool portdev_removed(rtnl_link *link);

src/netlink/nl_bond.cc

@@ -142,6 +142,8 @@ int nl_bond::add_lag(rtnl_link *bond) {
       swi->lag_remove(lag_id);
   }
 
+  nl->add_termination_mac(bond);
+
 #endif
 
   return rv;
@@ -158,6 +160,8 @@ int nl_bond::remove_lag(rtnl_link *bond) {
     return -ENODEV;
   }
 
+  nl->remove_termination_mac(bond);
+
   rv = swi->lag_remove(it->second);
   if (rv < 0) {
     LOG(ERROR) << __FUNCTION__