diff --git a/pkg/systemd/sysconfig.template b/pkg/systemd/sysconfig.template index fbc1e4eb..63760b29 100644 --- a/pkg/systemd/sysconfig.template +++ b/pkg/systemd/sysconfig.template @@ -9,8 +9,11 @@ # gRPC listening port: # FLAGS_ofdpa_grpc_port=50051 # -# Use KNET interfaces (experimental): -# FLAGS_use_knet=false +# Use KNET interfaces: +# FLAGS_use_knet=true +# +# Mark switched packets as offloaded: +# FLAGS_mark_fwd_offload=true ### glog # diff --git a/src/baseboxd.cc b/src/baseboxd.cc index 33aad7bd..60f498c6 100644 --- a/src/baseboxd.cc +++ b/src/baseboxd.cc @@ -19,7 +19,8 @@ DECLARE_string(tryfromenv); // from gflags DEFINE_bool(multicast, true, "Enable multicast support"); DEFINE_int32(port, 6653, "Listening port"); DEFINE_int32(ofdpa_grpc_port, 50051, "Listening port of ofdpa gRPC server"); -DEFINE_bool(use_knet, false, "Use KNET interfaces (experimental)"); +DEFINE_bool(use_knet, true, "Use KNET interfaces"); +DEFINE_bool(mark_fwd_offload, true, "Mark switched packets as offloaded"); static bool validate_port(const char *flagname, gflags::int32 value) { VLOG(3) << __FUNCTION__ << ": flagname=" << flagname << ", value=" << value; @@ -48,7 +49,8 @@ int main(int argc, char **argv) { } // all variables can be set from env - FLAGS_tryfromenv = std::string("multicast,port,ofdpa_grpc_port,use_knet"); + FLAGS_tryfromenv = + std::string("multicast,port,ofdpa_grpc_port,use_knet,mark_fwd_offload"); gflags::SetUsageMessage(""); gflags::SetVersionString(PROJECT_VERSION); diff --git a/src/netlink/cnetlink.cc b/src/netlink/cnetlink.cc index 6221d4cc..fce01c7f 100644 --- a/src/netlink/cnetlink.cc +++ b/src/netlink/cnetlink.cc @@ -42,6 +42,7 @@ #include "nl_vxlan.h" DECLARE_bool(multicast); +DECLARE_bool(mark_fwd_offload); namespace basebox { @@ -567,6 +568,8 @@ int cnetlink::add_l3_configuration(rtnl_link *link) { // add all ip addresses and routes from collected interfaces for (auto l : links) { + add_termination_mac(l); + rv = add_l3_addresses(l); if (rv < 0) LOG(WARNING) << __FUNCTION__ << ": failed to add l3 addresses (" << rv @@ -600,11 +603,44 @@ int cnetlink::remove_l3_configuration(rtnl_link *link) { if (rv < 0) LOG(WARNING) << __FUNCTION__ << ": failed to remove l3 addresses (" << rv << " from link " << OBJ_CAST(l); + remove_termination_mac(l); } return rv; } +int cnetlink::add_termination_mac(rtnl_link *link) { + struct nl_addr *addr = rtnl_link_get_addr(link); + auto mac = rofl::caddress_ll((uint8_t *)nl_addr_get_binary_addr(addr), + nl_addr_get_len(addr)); + uint32_t port_id = get_port_id(link); + uint16_t vid = 0; + + if (rtnl_link_is_vlan(link)) + vid = rtnl_link_vlan_get_id(link); + + swi->l3_termination_add(port_id, vid, mac); + swi->l3_termination_add_v6(port_id, vid, mac); + + return 0; +} + +int cnetlink::remove_termination_mac(rtnl_link *link) { + struct nl_addr *addr = rtnl_link_get_addr(link); + auto mac = rofl::caddress_ll((uint8_t *)nl_addr_get_binary_addr(addr), + nl_addr_get_len(addr)); + uint32_t port_id = get_port_id(link); + uint16_t vid = 0; + + if (rtnl_link_is_vlan(link)) + vid = rtnl_link_vlan_get_id(link); + + swi->l3_termination_remove_v6(port_id, vid, mac); + swi->l3_termination_remove(port_id, vid, mac); + + return 0; +} + int cnetlink::update_on_mac_change(rtnl_link *old_link, rtnl_link *new_link) { int rv = 0; int port_id = get_port_id(old_link); @@ -612,10 +648,8 @@ int cnetlink::update_on_mac_change(rtnl_link *old_link, rtnl_link *new_link) { struct nl_addr *old_mac = rtnl_link_get_addr(old_link); struct nl_addr *new_mac = rtnl_link_get_addr(new_link); - rv = l3->update_l3_termination(port_id, vid, old_mac, new_mac); - if (rv < 0) - VLOG(1) << __FUNCTION__ << ": failed to update termination MAC, old link=" - << OBJ_CAST(old_link) << " new link=" << OBJ_CAST(new_link); + remove_termination_mac(old_link); + add_termination_mac(new_link); // In response to the MAC address change on the interface, linux deletes the // neighbors configured on the interface. We are tracking the state @@ -1342,6 +1376,8 @@ void cnetlink::link_created(rtnl_link *link) noexcept { VLOG(1) << __FUNCTION__ << ": new vlan interface " << OBJ_CAST(link); uint16_t vid = rtnl_link_vlan_get_id(link); vlan->add_vlan(link, vid, true); + if (is_switch_interface(link)) + add_termination_mac(link); } break; case LT_BOND: { VLOG(1) << __FUNCTION__ << ": new bond interface " << OBJ_CAST(link); @@ -1352,9 +1388,16 @@ void cnetlink::link_created(rtnl_link *link) noexcept { } break; default: { bool handled = port_man->portdev_ready(link); - if (!handled) + if (handled) { + uint32_t port_id = get_port_id(link); + + port_man->set_offloaded(link, FLAGS_mark_fwd_offload); + swi->port_set_config(port_id, port_man->get_hwaddr(port_id), false); + add_termination_mac(link); + } else { LOG(WARNING) << __FUNCTION__ << ": ignoring link with lt=" << lt << " link:" << OBJ_CAST(link); + } } break; } // switch link type } @@ -1399,12 +1442,18 @@ void cnetlink::link_updated(rtnl_link *old_link, rtnl_link *new_link) noexcept { return; } + uint32_t port_id = port_man->get_port_id(rtnl_link_get_ifindex(new_link)); + if (port_id > 0 && (rtnl_link_get_flags(old_link) & IFF_UP) != + (rtnl_link_get_flags(new_link) & IFF_UP)) { + swi->port_set_config(port_id, port_man->get_hwaddr(port_id), + !!(rtnl_link_get_flags(new_link) & IFF_UP)); + } + switch (lt_old) { case LT_BOND_SLAVE: if (lt_new == LT_BOND_SLAVE) { // bond slave updated bond->update_lag_member(old_link, new_link); - } else if (port_man->get_port_id(rtnl_link_get_ifindex(new_link)) > - 0) { // bond slave removed + } else if (port_id > 0) { // bond slave removed bond->remove_lag_member(old_link); } break; @@ -1465,7 +1514,7 @@ void cnetlink::link_updated(rtnl_link *old_link, rtnl_link *new_link) noexcept { << ", new link: " << OBJ_CAST(new_link); break; default: - if (port_man->get_port_id(rtnl_link_get_ifindex(new_link)) > 0) { + if (port_id > 0) { if (lt_new == LT_BOND_SLAVE) { // XXX link enslaved LOG(INFO) << __FUNCTION__ << ": link enslaved " @@ -1521,6 +1570,7 @@ void cnetlink::link_deleted(rtnl_link *link) noexcept { bridge->clear_tpid_entries(); // clear the Egress TPID table delete bridge; bridge = nullptr; + remove_termination_mac(link); } break; case LT_VXLAN: { @@ -1537,6 +1587,8 @@ void cnetlink::link_deleted(rtnl_link *link) noexcept { case LT_VLAN: VLOG(1) << __FUNCTION__ << ": removed vlan interface " << OBJ_CAST(link); vlan->remove_vlan(link, rtnl_link_vlan_get_id(link), true); + if (is_switch_interface(link)) + remove_termination_mac(link); break; case LT_BOND: { VLOG(1) << __FUNCTION__ << ": removed bond interface " << OBJ_CAST(link); diff --git a/src/netlink/cnetlink.h b/src/netlink/cnetlink.h index a3489e41..cdd42415 100644 --- a/src/netlink/cnetlink.h +++ b/src/netlink/cnetlink.h @@ -66,6 +66,9 @@ class cnetlink final : public rofl::cthread_env { int add_l3_configuration(rtnl_link *link); int remove_l3_configuration(rtnl_link *link); + int add_termination_mac(rtnl_link *link); + int remove_termination_mac(rtnl_link *link); + int update_on_mac_change(rtnl_link *old_link, rtnl_link *new_link); bool has_l3_addresses(rtnl_link *link); diff --git a/src/netlink/knet_manager.cc b/src/netlink/knet_manager.cc index 7a5d6c13..59e8931b 100644 --- a/src/netlink/knet_manager.cc +++ b/src/netlink/knet_manager.cc @@ -84,8 +84,15 @@ int knet_manager::create_portdev(uint32_t port_id, const std::string &port_name, auto rv = port_names2id.emplace(std::make_pair(port_name, port_id)); if (!rv.second) { - LOG(FATAL) << __FUNCTION__ << ": failed to insert"; + LOG(FATAL) << __FUNCTION__ << ": failed to insert port name"; } + + auto rv2 = id_to_hwaddr.emplace(std::make_pair(port_id, hwaddr)); + + if (!rv2.second) { + LOG(FATAL) << __FUNCTION__ << ": failed to insert hwaddr"; + } + r = system(("/usr/sbin/client_drivshell knet netif create port=" + std::to_string(port_id) + " ifname=" + port_name + " mac=" + mac_string + " keeprxtag=yes") @@ -313,4 +320,15 @@ int knet_manager::set_port_speed(const std::string name, uint32_t speed, return 1; } +int knet_manager::set_offloaded(rtnl_link *link, bool offloaded) { + std::string name(rtnl_link_get_name(link)); + std::ofstream file("/proc/bcm/knet/link"); + + if (file.is_open()) { + file << (name + (offloaded ? "=offload" : "=no-offload")); + file.close(); + } + return 0; +} + } // namespace basebox diff --git a/src/netlink/knet_manager.h b/src/netlink/knet_manager.h index 1159f8a0..a22a5937 100644 --- a/src/netlink/knet_manager.h +++ b/src/netlink/knet_manager.h @@ -37,6 +37,7 @@ class knet_manager final : public port_manager { int change_port_status(const std::string name, bool status); int set_port_speed(const std::string name, uint32_t speed, uint8_t duplex); + int set_offloaded(rtnl_link *link, bool offloaded); // access from northbound (cnetlink) bool portdev_removed(rtnl_link *link); diff --git a/src/netlink/nl_bond.cc b/src/netlink/nl_bond.cc index ab33a92b..84c2ffeb 100644 --- a/src/netlink/nl_bond.cc +++ b/src/netlink/nl_bond.cc @@ -142,6 +142,8 @@ int nl_bond::add_lag(rtnl_link *bond) { swi->lag_remove(lag_id); } + nl->add_termination_mac(bond); + #endif return rv; @@ -158,6 +160,8 @@ int nl_bond::remove_lag(rtnl_link *bond) { return -ENODEV; } + nl->remove_termination_mac(bond); + rv = swi->lag_remove(it->second); if (rv < 0) { LOG(ERROR) << __FUNCTION__ diff --git a/src/netlink/nl_l3.cc b/src/netlink/nl_l3.cc index 68a824ab..d5f666f8 100644 --- a/src/netlink/nl_l3.cc +++ b/src/netlink/nl_l3.cc @@ -69,10 +69,6 @@ std::unordered_map< l3_interface> l3_interface_mapping; -// key: source port_id, vid, src_mac, af ; value: refcount -std::unordered_set> - termination_mac_entries; - // ECMP mapping std::unordered_multimap, l3_interface> l3_ecmp_mapping; @@ -191,21 +187,6 @@ int nl_l3::add_l3_addr(struct rtnl_addr *a) { return -EINVAL; } - // XXX TODO split this into several functions - if (!is_loopback) { - int port_id = nl->get_port_id(link); - auto addr = rtnl_link_get_addr(link); - rofl::caddress_ll mac = libnl_lladdr_2_rofl(addr); - - rv = add_l3_termination(port_id, vid, mac, AF_INET); - if (rv < 0) { - LOG(ERROR) << __FUNCTION__ - << ": failed to setup termination mac port_id=" << port_id - << ", vid=" << vid << " mac=" << mac << "; rv=" << rv; - return rv; - } - } - // get v4 dst (local v4 addr) auto prefixlen = rtnl_addr_get_prefixlen(a); auto addr = rtnl_addr_get_local(a); @@ -213,7 +194,6 @@ int nl_l3::add_l3_addr(struct rtnl_addr *a) { rofl::caddress_in4 mask = rofl::build_mask_in4(prefixlen); if (rv < 0) { - // TODO shall we remove the l3_termination mac? LOG(ERROR) << __FUNCTION__ << ": could not parse addr " << addr; return rv; } @@ -248,7 +228,6 @@ int nl_l3::add_l3_addr(struct rtnl_addr *a) { if (prefixlen == 32) { rv = sw->l3_unicast_host_add(ipv4_dst, 0, false, update, vrf_id); if (rv < 0) { - // TODO shall we remove the l3_termination mac? LOG(ERROR) << __FUNCTION__ << ": failed to setup l3 addr " << addr; } } @@ -299,20 +278,6 @@ int nl_l3::add_l3_addr_v6(struct rtnl_addr *a) { uint16_t vid = vlan->get_vid(link); - if (!is_loopback) { - int port_id = nl->get_port_id(link); - auto addr = rtnl_link_get_addr(link); - rofl::caddress_ll mac = libnl_lladdr_2_rofl(addr); - - rv = add_l3_termination(port_id, vid, mac, AF_INET6); - if (rv < 0) { - LOG(ERROR) << __FUNCTION__ - << ": failed to setup termination mac port_id=" << port_id - << ", vid=" << vid << " mac=" << mac << "; rv=" << rv; - return rv; - } - } - if (is_loopback) { rv = add_lo_addr_v6(a); return rv; @@ -491,20 +456,6 @@ int nl_l3::del_l3_addr(struct rtnl_addr *a) { get_l3_addrs(other, &addresses, family); } - if (addresses.empty()) { - int port_id = nl->get_port_id(link); - - addr = rtnl_link_get_addr(link); - rofl::caddress_ll mac = libnl_lladdr_2_rofl(addr); - - rv = del_l3_termination(port_id, vid, mac, family); - if (rv < 0 && rv != -ENODATA) { - LOG(ERROR) << __FUNCTION__ - << ": failed to remove l3 termination mac(local) vid=" << vid - << "; rv=" << rv; - } - } - // del vlan // Avoid deleting table VLAN entry for the following two cases // Loopback: does not require entry on the Ingress table @@ -1223,133 +1174,6 @@ int nl_l3::del_l3_route(struct rtnl_route *r) { } } -int nl_l3::add_l3_termination(uint32_t port_id, uint16_t vid, - const rofl::caddress_ll &mac, int af) noexcept { - int rv = 0; - - // lookup if this already exists - auto needle = std::make_tuple(port_id, vid, mac, static_cast(af)); - auto it = termination_mac_entries.find(needle); - if (it != termination_mac_entries.end()) - return 0; - - termination_mac_entries.emplace(std::move(needle)); - - switch (af) { - case AF_INET: - rv = sw->l3_termination_add(port_id, vid, mac); - break; - - case AF_INET6: - rv = sw->l3_termination_add_v6(port_id, vid, mac); - break; - - default: - LOG(FATAL) << __FUNCTION__ << ": invalid address family " << af; - break; - } - - if (rv == 0) - - VLOG(3) << __FUNCTION__ << ": added l3 termination for port=" << port_id - << " vid=" << vid << " mac=" << mac << " af=" << af; - - return rv; -} - -int nl_l3::del_l3_termination(uint32_t port_id, uint16_t vid, - const rofl::caddress_ll &mac, int af) noexcept { - int rv = 0; - - VLOG(4) << __FUNCTION__ << ": trying to delete for port_id=" << port_id - << ", vid=" << vid << ", mac=" << mac << ", af=" << af; - - // lookup if this exists - auto needle = std::make_tuple(port_id, vid, mac, static_cast(af)); - auto it = termination_mac_entries.find(needle); - if (it == termination_mac_entries.end()) { - LOG(WARNING) - << __FUNCTION__ - << ": tried to delete a non existing termination mac for port_id=" - << port_id << ", vid=" << vid << ", mac=" << mac << ", af=" << af; - return -ENODATA; - } - - switch (af) { - case AF_INET: - rv = sw->l3_termination_remove(port_id, vid, mac); - break; - - case AF_INET6: - rv = sw->l3_termination_remove_v6(port_id, vid, mac); - break; - - default: - LOG(FATAL) << __FUNCTION__ << ": invalid address family " << af; - break; - } - - termination_mac_entries.erase(it); - - return rv; -} - -int nl_l3::update_l3_termination(int port_id, uint16_t vid, - struct nl_addr *old_mac, - struct nl_addr *new_mac) noexcept { - int rv = 0; - - auto o_mac = libnl_lladdr_2_rofl(old_mac); - auto n_mac = libnl_lladdr_2_rofl(new_mac); - - // parse the AF list and remove the entry from the termination mac set - // call the switch function to remove and insert the entry with the - // new mac address. - if (termination_mac_entries.find(std::make_tuple( - port_id, vid, o_mac, AF_INET)) != termination_mac_entries.end()) { - rv = del_l3_termination(port_id, vid, o_mac, AF_INET); - if (rv < 0) - VLOG(3) << __FUNCTION__ - << ": failed to remove termination mac port=" << port_id - << " vid=" << vid << " mac=" << o_mac; - rv = add_l3_termination(port_id, vid, n_mac, AF_INET); - if (rv < 0) { - VLOG(3) << __FUNCTION__ - << ": failed to add termination mac port=" << port_id - << " vid=" << vid << " mac=" << n_mac; - return rv; - } - - VLOG(2) << __FUNCTION__ - << ": updated Termination MAC for port_id=" << port_id - << " old mac address=" << o_mac << " new mac address=" << n_mac - << " AF=" << AF_INET; - } - - if (termination_mac_entries.find(std::make_tuple( - port_id, vid, o_mac, AF_INET6)) != termination_mac_entries.end()) { - rv = del_l3_termination(port_id, vid, o_mac, AF_INET6); - if (rv < 0) - VLOG(3) << __FUNCTION__ - << ": failed to remove termination mac port=" << port_id - << " vid=" << vid << " mac=" << o_mac; - rv = add_l3_termination(port_id, vid, n_mac, AF_INET6); - if (rv < 0) { - VLOG(3) << __FUNCTION__ - << ": failed to add termination mac port=" << port_id - << " vid=" << vid << " mac=" << n_mac; - return rv; - } - - VLOG(2) << __FUNCTION__ - << ": updated Termination MAC for port_id=" << port_id - << " old mac address=" << o_mac << " new mac address=" << n_mac - << " AF=" << AF_INET6; - } - - return rv; -} - int nl_l3::update_l3_egress(int port_id, uint16_t vid, struct nl_addr *old_mac, struct nl_addr *new_mac) noexcept { diff --git a/src/netlink/port_manager.h b/src/netlink/port_manager.h index 4400cb8f..510afc18 100644 --- a/src/netlink/port_manager.h +++ b/src/netlink/port_manager.h @@ -68,15 +68,27 @@ class port_manager { } } + const rofl::caddress_ll get_hwaddr(uint32_t port_id) const noexcept { + // XXX TODO add assert wrt threading + auto it = id_to_hwaddr.find(port_id); + if (it == id_to_hwaddr.end()) { + return nulladdr; + } else { + return it->second; + } + } + void clear() noexcept { std::lock_guard lock(tn_mutex); ifindex_to_id.clear(); id_to_ifindex.clear(); + id_to_hwaddr.clear(); } virtual int change_port_status(const std::string name, bool status) = 0; virtual int set_port_speed(const std::string name, uint32_t speed, uint8_t duplex) = 0; + virtual int set_offloaded(rtnl_link *link, bool offloaded) = 0; // access from northbound (cnetlink) virtual bool portdev_removed(rtnl_link *link) = 0; @@ -94,6 +106,9 @@ class port_manager { // only accessible from cnetlink std::map ifindex_to_id; std::map id_to_ifindex; + std::map id_to_hwaddr; + + const rofl::caddress_ll nulladdr; }; } // namespace basebox diff --git a/src/netlink/tap_manager.cc b/src/netlink/tap_manager.cc index 4b408d2a..30a3c0be 100644 --- a/src/netlink/tap_manager.cc +++ b/src/netlink/tap_manager.cc @@ -64,7 +64,13 @@ int tap_manager::create_portdev(uint32_t port_id, const std::string &port_name, auto rv = port_names2id.emplace(std::make_pair(port_name, port_id)); if (!rv.second) { - LOG(FATAL) << __FUNCTION__ << ": failed to insert"; + LOG(FATAL) << __FUNCTION__ << ": failed to insert port name"; + } + + auto rv2 = id_to_hwaddr.emplace(std::make_pair(port_id, hwaddr)); + + if (!rv2.second) { + LOG(FATAL) << __FUNCTION__ << ": failed to insert hwaddr"; } dev->tap_open(); @@ -425,4 +431,6 @@ int tap_manager::set_port_speed(const std::string name, uint32_t speed, return error; } +int tap_manager::set_offloaded(rtnl_link *link, bool offloaded) { return 0; } + } // namespace basebox diff --git a/src/netlink/tap_manager.h b/src/netlink/tap_manager.h index fd8962af..f1e1546c 100644 --- a/src/netlink/tap_manager.h +++ b/src/netlink/tap_manager.h @@ -40,6 +40,7 @@ class tap_manager final : public port_manager { int change_port_status(const std::string name, bool status); int set_port_speed(const std::string name, uint32_t speed, uint8_t duplex); + int set_offloaded(rtnl_link *link, bool offloaded); // access from northbound (cnetlink) bool portdev_removed(rtnl_link *link); diff --git a/src/of-dpa/controller.cc b/src/of-dpa/controller.cc index b4884d18..9cda6edb 100644 --- a/src/of-dpa/controller.cc +++ b/src/of-dpa/controller.cc @@ -2042,6 +2042,28 @@ int controller::delete_egress_tpid(uint32_t port) noexcept { } return rv; } + +int controller::port_set_config(uint32_t port, const rofl::caddress_ll &mac, + bool up) noexcept { + int rv = 0; + try { + rofl::crofdpt &dpt = set_dpt(dptid, true); + dpt.send_port_mod_message(rofl::cauxid(0), port, mac, + up ? 0 : rofl::openflow13::OFPPC_PORT_DOWN, + rofl::openflow13::OFPPC_PORT_DOWN, 0); + } catch (rofl::eRofBaseNotFound &e) { + LOG(ERROR) << ": caught rofl::eRofBaseNotFound"; + rv = -EINVAL; + } catch (rofl::eRofConnNotConnected &e) { + LOG(ERROR) << ": not connected msg=" << e.what(); + rv = -ENOTCONN; + } catch (std::exception &e) { + LOG(ERROR) << ": caught unknown exception: " << e.what(); + rv = -EINVAL; + } + return rv; +} + int controller::subscribe_to(enum swi_flags flags) noexcept { int rv = 0; this->flags = this->flags | flags; diff --git a/src/of-dpa/controller.h b/src/of-dpa/controller.h index 5e548d00..612adc7b 100644 --- a/src/of-dpa/controller.h +++ b/src/of-dpa/controller.h @@ -267,6 +267,9 @@ class controller : public rofl::crofbase, const sai_port_stat_t *counter_ids, uint64_t *counters) noexcept override; + int port_set_config(uint32_t port_id, const rofl::caddress_ll &mac, + bool up) noexcept override; + /* IO */ int enqueue(uint32_t port_id, basebox::packet *pkt) noexcept override; diff --git a/src/sai.h b/src/sai.h index 0dc741c4..3dcea96f 100644 --- a/src/sai.h +++ b/src/sai.h @@ -183,6 +183,11 @@ class switch_interface { virtual int delete_egress_tpid(uint32_t port) noexcept = 0; /* @} */ + /* @ port { */ + virtual int port_set_config(uint32_t port_id, const rofl::caddress_ll &mac, + bool up) noexcept = 0; + /* @} */ + /* @ control { */ virtual int enqueue(uint32_t port_id, basebox::packet *pkt) noexcept = 0; virtual int subscribe_to(enum swi_flags flags) noexcept = 0;