From 0028f4fff8f1d5eab9c0134fd5162ece34f424cf Mon Sep 17 00:00:00 2001 From: Rajesh Sankaran <48232228+srj102@users.noreply.github.com> Date: Mon, 14 Dec 2020 23:03:06 +0530 Subject: [PATCH] [VxlanMgr] changes for EVPN VXLAN (#1266) * VxlanMgr changes for EVPN VXLAN 1. Add support for EVPN NVO config table. 2. VxlanTunnel and TunnelMap handlers are enhanced to also cache the data so that this can be used during the deletion. 3. ARP suppression changes 4. Create state db for vxlanmgr and vlanmgr to sync the tunnel association to vlan 5. Changes to support warm reboot. Signed-off-by: Rajesh Sankaran --- cfgmgr/vxlanmgr.cpp | 671 ++++++++++++++++++++++++++++++++++++++-- cfgmgr/vxlanmgr.h | 47 ++- cfgmgr/vxlanmgrd.cpp | 61 ++++ orchagent/portsorch.cpp | 2 +- orchagent/vxlanorch.cpp | 2 +- 5 files changed, 746 insertions(+), 37 deletions(-) diff --git a/cfgmgr/vxlanmgr.cpp b/cfgmgr/vxlanmgr.cpp index cdb3c2ab8005..be570dff5763 100644 --- a/cfgmgr/vxlanmgr.cpp +++ b/cfgmgr/vxlanmgr.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -16,6 +17,8 @@ using namespace std; using namespace swss; +extern MacAddress gMacAddress; + // Fields name #define VXLAN_TUNNEL "vxlan_tunnel" #define SOURCE_IP "src_ip" @@ -30,6 +33,10 @@ using namespace swss; #define VXLAN_NAME_PREFIX "Vxlan" #define VXLAN_IF_NAME_PREFIX "Brvxlan" +#define VLAN "vlan" +#define DST_IP "dst_ip" +#define SOURCE_VTEP "source_vtep" + static std::string getVxlanName(const swss::VxlanMgr::VxlanInfo & info) { return std::string("") + VXLAN_NAME_PREFIX + info.m_vni; @@ -165,17 +172,28 @@ static int cmdDetachVxlanIfFromVnet(const swss::VxlanMgr::VxlanInfo & info, std: // Vxlanmgr VxlanMgr::VxlanMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, const vector &tables) : + m_app_db(appDb), Orch(cfgDb, tables), m_appVxlanTunnelTable(appDb, APP_VXLAN_TUNNEL_TABLE_NAME), m_appVxlanTunnelMapTable(appDb, APP_VXLAN_TUNNEL_MAP_TABLE_NAME), m_appSwitchTable(appDb, APP_SWITCH_TABLE_NAME), + m_appEvpnNvoTable(appDb, APP_VXLAN_EVPN_NVO_TABLE_NAME), m_cfgVxlanTunnelTable(cfgDb, CFG_VXLAN_TUNNEL_TABLE_NAME), m_cfgVnetTable(cfgDb, CFG_VNET_TABLE_NAME), m_stateVrfTable(stateDb, STATE_VRF_TABLE_NAME), - m_stateVxlanTable(stateDb, STATE_VXLAN_TABLE_NAME) + m_stateVxlanTable(stateDb, STATE_VXLAN_TABLE_NAME), + m_stateVlanTable(stateDb, STATE_VLAN_TABLE_NAME), + m_stateTunnelVlanMapTable(stateDb, STATE_NEIGH_SUPPRESS_VLAN_TABLE_NAME), + m_stateVxlanTunnelTable(stateDb, STATE_VXLAN_TUNNEL_TABLE_NAME) { - // Clear old vxlan devices that were created at last time. - clearAllVxlanDevices(); + getAllVxlanNetDevices(); + + if (!WarmStart::isWarmStart()) + { + // Clear old vxlan devices that were created at last time. + clearAllVxlanDevices(); + } + } VxlanMgr::~VxlanMgr() @@ -209,6 +227,10 @@ void VxlanMgr::doTask(Consumer &consumer) { task_result = doVxlanTunnelMapCreateTask(t); } + else if (table_name == CFG_VXLAN_EVPN_NVO_TABLE_NAME) + { + task_result = doVxlanEvpnNvoCreateTask(t); + } else { SWSS_LOG_ERROR("Unknown table : %s", table_name.c_str()); @@ -228,6 +250,10 @@ void VxlanMgr::doTask(Consumer &consumer) { task_result = doVxlanTunnelMapDeleteTask(t); } + else if (table_name == CFG_VXLAN_EVPN_NVO_TABLE_NAME) + { + task_result = doVxlanEvpnNvoDeleteTask(t); + } else { SWSS_LOG_ERROR("Unknown table : %s", table_name.c_str()); @@ -285,7 +311,7 @@ bool VxlanMgr::doVxlanCreateTask(const KeyOpFieldsValuesTuple & t) if (it == m_vxlanTunnelCache.end()) { SWSS_LOG_DEBUG("Vxlan tunnel %s has not been created", info.m_vxlanTunnel.c_str()); - // Suspend this message util the vxlan tunnel is created + // Suspend this message until the vxlan tunnel is created return false; } @@ -293,7 +319,7 @@ bool VxlanMgr::doVxlanCreateTask(const KeyOpFieldsValuesTuple & t) if (!isVrfStateOk(info.m_vnet)) { SWSS_LOG_DEBUG("Vrf %s has not been created", info.m_vnet.c_str()); - // Suspend this message util the vrf is created + // Suspend this message until the vrf is created return false; } @@ -302,16 +328,16 @@ bool VxlanMgr::doVxlanCreateTask(const KeyOpFieldsValuesTuple & t) if (!macAddress.first) { SWSS_LOG_DEBUG("Mac address is not ready"); - // Suspend this message util the mac address is set + // Suspend this message until the mac address is set return false; } info.m_macAddress = macAddress.second; auto sourceIp = std::find_if( - it->second.begin(), - it->second.end(), + it->second.fvt.begin(), + it->second.fvt.end(), [](const FieldValueTuple & fvt){ return fvt.first == SOURCE_IP; }); - if (sourceIp != it->second.end()) + if (sourceIp != it->second.fvt.end()) { info.m_sourceIp = sourceIp->second; } @@ -369,7 +395,6 @@ bool VxlanMgr::doVxlanDeleteTask(const KeyOpFieldsValuesTuple & t) m_vnetCache.erase(it); SWSS_LOG_INFO("Delete vxlan %s", info.m_vxlan.c_str()); return true; - } bool VxlanMgr::doVxlanTunnelCreateTask(const KeyOpFieldsValuesTuple & t) @@ -377,12 +402,28 @@ bool VxlanMgr::doVxlanTunnelCreateTask(const KeyOpFieldsValuesTuple & t) SWSS_LOG_ENTER(); const std::string & vxlanTunnelName = kfvKey(t); - m_appVxlanTunnelTable.set(vxlanTunnelName, kfvFieldsValues(t)); // Update vxlan tunnel cache - m_vxlanTunnelCache[vxlanTunnelName] = kfvFieldsValues(t); + TunCache tuncache; + + tuncache.fvt = kfvFieldsValues(t); + tuncache.vlan_vni_refcnt = 0; + tuncache.m_sourceIp = "NULL"; - SWSS_LOG_INFO("Create vxlan tunnel %s", vxlanTunnelName.c_str()); + for (auto i : kfvFieldsValues(t)) + { + const std::string & field = fvField(i); + const std::string & value = fvValue(i); + if (field == SOURCE_IP) + { + tuncache.m_sourceIp = value; + } + } + + m_appVxlanTunnelTable.set(vxlanTunnelName, kfvFieldsValues(t)); + m_vxlanTunnelCache[vxlanTunnelName] = tuncache; + + SWSS_LOG_NOTICE("Create vxlan tunnel %s", vxlanTunnelName.c_str()); return true; } @@ -391,27 +432,182 @@ bool VxlanMgr::doVxlanTunnelDeleteTask(const KeyOpFieldsValuesTuple & t) SWSS_LOG_ENTER(); const std::string & vxlanTunnelName = kfvKey(t); - m_appVxlanTunnelTable.del(vxlanTunnelName); - auto it = m_vxlanTunnelCache.find(vxlanTunnelName); - if (it != m_vxlanTunnelCache.end()) + // If there is an NVO referring to this tunnel then hold on. + std::map::iterator it = m_EvpnNvoCache.begin(); + + if ((it != m_EvpnNvoCache.end()) && (it->second == vxlanTunnelName)) + { + SWSS_LOG_WARN("Tunnel %s deletion failed. Need to delete NVO", vxlanTunnelName.c_str()); + return false; + } + + // If there are mappings still against this tunnel then hold on. + if (m_vxlanTunnelCache[vxlanTunnelName].vlan_vni_refcnt) + { + SWSS_LOG_WARN("Tunnel %s deletion failed. Need to delete mapping entries", vxlanTunnelName.c_str()); + return false; + } + + if (isTunnelActive(vxlanTunnelName)) + { + m_appVxlanTunnelTable.del(vxlanTunnelName); + } + + auto it1 = m_vxlanTunnelCache.find(vxlanTunnelName); + if (it1 != m_vxlanTunnelCache.end()) { - m_vxlanTunnelCache.erase(it); + m_vxlanTunnelCache.erase(it1); } - SWSS_LOG_INFO("Delete vxlan tunnel %s", vxlanTunnelName.c_str()); + SWSS_LOG_NOTICE("Delete vxlan tunnel %s", vxlanTunnelName.c_str()); return true; } bool VxlanMgr::doVxlanTunnelMapCreateTask(const KeyOpFieldsValuesTuple & t) { + int ret; SWSS_LOG_ENTER(); std::string vxlanTunnelMapName = kfvKey(t); std::replace(vxlanTunnelMapName.begin(), vxlanTunnelMapName.end(), config_db_key_delimiter, delimiter); - m_appVxlanTunnelMapTable.set(vxlanTunnelMapName, kfvFieldsValues(t)); - SWSS_LOG_NOTICE("Create vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + if (m_vxlanTunnelMapCache.find(vxlanTunnelMapName) != m_vxlanTunnelMapCache.end()) + { + SWSS_LOG_ERROR("Map already present : %s", vxlanTunnelMapName.c_str()); + return true; + } + + SWSS_LOG_INFO("Create vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + std::string vlan, vlan_id, vni_id, src_ip, dst_ip(""); + for (auto i : kfvFieldsValues(t)) + { + const std::string & field = fvField(i); + const std::string & value = fvValue(i); + if (field == VLAN) + { + vlan = value; + } + else if (field == VNI) + { + vni_id = value; + } + } + + // Check for VLAN or VNI if they are already mapped + if (m_vlanMapCache.find(vlan) != m_vlanMapCache.end()) + { + SWSS_LOG_ERROR("Vlan %s already mapped. Map Create failed for : %s", + vlan.c_str(), vxlanTunnelMapName.c_str()); + return true; + } + + if (m_vniMapCache.find(vni_id) != m_vniMapCache.end()) + { + SWSS_LOG_ERROR("VNI %s already mapped. Map Create failed for : %s", + vni_id.c_str(), vxlanTunnelMapName.c_str()); + return true; + } + + const auto vlan_prefix = std::string("Vlan"); + const auto prefix_len = vlan_prefix.length(); + vlan_id = vlan.substr(prefix_len); + + size_t found = vxlanTunnelMapName.find(delimiter); + const auto vxlanTunnelName = vxlanTunnelMapName.substr(0, found); + + // If the vxlan tunnel has been created + auto it = m_vxlanTunnelCache.find(vxlanTunnelName); + if (!isTunnelActive(vxlanTunnelName)) + { + SWSS_LOG_INFO("Vxlan tunnel %s has not been created", vxlanTunnelName.c_str()); + // Suspend this message until the vxlan tunnel is created + return false; + } + + if (!isVlanStateOk(vlan)) + { + SWSS_LOG_INFO("VLAN id is not yet created : %s",vxlanTunnelMapName.c_str()); + return false; + } + + // Check the below condition only after the vxlanmgrd has reached reconcile state + // The check to verify the state vxlan table is to take care of back to back + // create and delete of a VTEP object. On deletion of a VTEP object the FRR takes + // some time to remove all the routes and once all the routes are removed, the p2p + // tunnel is also removed. This check waits for all the p2p tunnels which were associated + // with the earlier version of the VTEP to be deleted before processing further map entry + // creations. + WarmStart::WarmStartState state; + WarmStart::getWarmStartState("vxlanmgrd",state); + if (state == WarmStart::RECONCILED) + { + if (m_vxlanTunnelMapCache.empty()) + { + std::vector keys; + m_stateVxlanTunnelTable.getKeys(keys); + if (!keys.empty()) + { + SWSS_LOG_WARN("State VXLAN tunnel table not yet empty."); + return false; + } + } + } + + auto sourceIp = std::find_if( + it->second.fvt.begin(), + it->second.fvt.end(), + [](const FieldValueTuple & fvt){ return fvt.first == SOURCE_IP; }); + if (sourceIp == it->second.fvt.end()) + { + SWSS_LOG_DEBUG("Vxlan tunnel %s has no field src_ip", vxlanTunnelName.c_str()); + return true; + } + else + { + src_ip = sourceIp->second; + } + auto dstIp = std::find_if( + it->second.fvt.begin(), + it->second.fvt.end(), + [](const FieldValueTuple & fvt){ return fvt.first == DST_IP; }); + if (dstIp != it->second.fvt.end()) + { + dst_ip = dstIp->second; + } + else + { + dst_ip = ""; + } + + createAppDBTunnelMapTable(t); + ret = createVxlanNetdevice(vxlanTunnelName, vni_id, src_ip, dst_ip, vlan_id); + if (ret != RET_SUCCESS) + { + SWSS_LOG_WARN("Vxlan Net Dev creation failure for %s VNI(%s) VLAN(%s)", + vxlanTunnelName.c_str(), vni_id.c_str(), vlan_id.c_str()); + } + + std::string vxlan_dev_name; + vxlan_dev_name = std::string("") + std::string(vxlanTunnelName) + "-" + std::string(vlan_id); + + MapCache map_entry; + map_entry.vxlan_dev_name = vxlan_dev_name; + map_entry.vlan = vlan; + map_entry.vni_id = vni_id; + + m_vxlanTunnelMapCache[vxlanTunnelMapName] = map_entry; + m_vlanMapCache[vlan] = vni_id; + m_vniMapCache[vni_id] = vlan; + m_vxlanTunnelCache[vxlanTunnelName].vlan_vni_refcnt++; + + //Inform the Vlan Mgr to update the tunnel flags if Arp/Nd Suppression is set. + std::string key = "Vlan" + std::string(vlan_id); + vector fvVector; + FieldValueTuple s("netdev", vxlan_dev_name); + fvVector.push_back(s); + m_stateTunnelVlanMapTable.set(key,fvVector); + return true; } @@ -421,9 +617,110 @@ bool VxlanMgr::doVxlanTunnelMapDeleteTask(const KeyOpFieldsValuesTuple & t) std::string vxlanTunnelMapName = kfvKey(t); std::replace(vxlanTunnelMapName.begin(), vxlanTunnelMapName.end(), config_db_key_delimiter, delimiter); - m_appVxlanTunnelMapTable.del(vxlanTunnelMapName); - SWSS_LOG_NOTICE("Delete vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + delAppDBTunnelMapTable(vxlanTunnelMapName); + + SWSS_LOG_INFO("Delete vxlan tunnel map %s", vxlanTunnelMapName.c_str()); + + // ip link del dev {{VXLAN}} + size_t found = vxlanTunnelMapName.find(delimiter); + const auto vxlanTunnelName = vxlanTunnelMapName.substr(0, found); + + std::string vxlan_dev_name,vlan,vni_id; + MapCache map_entry; + + try + { + map_entry = m_vxlanTunnelMapCache.at(vxlanTunnelMapName); + } + catch (const std::out_of_range& oor) + { + SWSS_LOG_ERROR("Error deleting tunnmap : %s exception : %s", + vxlanTunnelMapName.c_str(), oor.what()); + return true; + } + + vxlan_dev_name = map_entry.vxlan_dev_name; + vlan = map_entry.vlan; + vni_id = map_entry.vni_id; + deleteVxlanNetdevice(vxlan_dev_name); + + m_vxlanTunnelMapCache.erase(vxlanTunnelMapName); + m_vlanMapCache.erase(vlan); + m_vniMapCache.erase(vni_id); + m_vxlanTunnelCache[vxlanTunnelName].vlan_vni_refcnt--; + + //Delete the state table map of vlan to tunnel name. + std::string vlan_delimiter = "-"; + found = vxlan_dev_name.find(vlan_delimiter); + std::string key = "Vlan" + vxlan_dev_name.substr(found+1,vxlan_dev_name.length()); + SWSS_LOG_INFO("Delete Tunnel Map for %s -> %s ", key.c_str(), vxlan_dev_name.c_str()); + m_stateTunnelVlanMapTable.del(key); + return true; +} + +bool VxlanMgr::doVxlanEvpnNvoCreateTask(const KeyOpFieldsValuesTuple & t) +{ + SWSS_LOG_ENTER(); + + std::string EvpnNvoName = kfvKey(t); + + if (m_EvpnNvoCache.find(EvpnNvoName) != m_EvpnNvoCache.end()) + { + SWSS_LOG_ERROR("Only Single NVO object allowed"); + return true; + } + + for (auto i : kfvFieldsValues(t)) + { + const std::string & field = fvField(i); + const std::string & value = fvValue(i); + if (!isTunnelActive(value)) + { + SWSS_LOG_ERROR("NVO %s creation failed. VTEP not present",EvpnNvoName.c_str()); + return false; + } + if (field == SOURCE_VTEP) + { + m_EvpnNvoCache[EvpnNvoName] = value; + } + } + + std::replace(EvpnNvoName.begin(), EvpnNvoName.end(), config_db_key_delimiter, delimiter); + m_appEvpnNvoTable.set(EvpnNvoName, kfvFieldsValues(t)); + + SWSS_LOG_INFO("Create evpn nvo %s", EvpnNvoName.c_str()); + return true; +} + +bool VxlanMgr::doVxlanEvpnNvoDeleteTask(const KeyOpFieldsValuesTuple & t) +{ + SWSS_LOG_ENTER(); + + std::string EvpnNvoName = kfvKey(t); + std::string vtep_name; + try + { + vtep_name = m_EvpnNvoCache.at(EvpnNvoName); + } + catch (const std::out_of_range& oor) + { + SWSS_LOG_ERROR("NVOdeletion NVO : %s not found exception : %s", EvpnNvoName.c_str(), oor.what()); + return true; + } + + // If there are mappings still then the NVO cannot be deleted. + if (m_vxlanTunnelCache[vtep_name].vlan_vni_refcnt) + { + return false; + } + + m_EvpnNvoCache.erase(EvpnNvoName); + + std::replace(EvpnNvoName.begin(), EvpnNvoName.end(), config_db_key_delimiter, delimiter); + m_appEvpnNvoTable.del(EvpnNvoName); + + SWSS_LOG_INFO("Delete evpn nvo %s", EvpnNvoName.c_str()); return true; } @@ -456,6 +753,23 @@ bool VxlanMgr::isVxlanStateOk(const std::string & vxlanName) return false; } +bool VxlanMgr::isVlanStateOk(const std::string &vlanName) +{ + SWSS_LOG_ENTER(); + std::vector temp; + + if (!vlanName.compare(0, strlen(VLAN_PREFIX), VLAN_PREFIX)) + { + if (m_stateVlanTable.get(vlanName, temp)) + { + SWSS_LOG_DEBUG("%s is ready", vlanName.c_str()); + return true; + } + } + SWSS_LOG_INFO("%s is not ready", vlanName.c_str()); + return false; +} + std::pair VxlanMgr::getVxlanRouterMacAddress() { std::vector temp; @@ -585,10 +899,142 @@ bool VxlanMgr::deleteVxlan(const VxlanInfo & info) return true; } -void VxlanMgr::clearAllVxlanDevices() +void VxlanMgr::createAppDBTunnelMapTable(const KeyOpFieldsValuesTuple & t) +{ + std::string vxlanTunnelMapName = kfvKey(t); + + std::replace(vxlanTunnelMapName.begin(), vxlanTunnelMapName.end(), config_db_key_delimiter, delimiter); + + /* Case 1: Entry exist - Erase from cache & return + * Case 2: Enry does not exist - Write to AppDB + * Case 3: Entry exist but modified - Not taken care. Will address later + */ + if (m_in_reconcile) + { + auto it = find(m_appVxlanTunnelMapKeysRecon.begin(), m_appVxlanTunnelMapKeysRecon.end(), vxlanTunnelMapName); + if (it != m_appVxlanTunnelMapKeysRecon.end()) + { + m_appVxlanTunnelMapKeysRecon.erase(it); + SWSS_LOG_INFO("Reconcile App Tunnel Map Table create %s reconciled. Pending %lu", + vxlanTunnelMapName.c_str(), m_appVxlanTunnelMapKeysRecon.size()); + return; + } + else + { + SWSS_LOG_INFO("Reconcile App Tunnel Map Table create %s doesnt not exist. Pending %lu", + vxlanTunnelMapName.c_str(), m_appVxlanTunnelMapKeysRecon.size()); + } + } + else + { + SWSS_LOG_INFO("App Tunnel Map Table create %s", vxlanTunnelMapName.c_str()); + } + m_appVxlanTunnelMapTable.set(vxlanTunnelMapName, kfvFieldsValues(t)); + + return; +} + +void VxlanMgr::delAppDBTunnelMapTable(std::string vxlanTunnelMapName) +{ + m_appVxlanTunnelMapTable.del(vxlanTunnelMapName); +} + +int VxlanMgr::createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_id, + std::string src_ip, std::string dst_ip, + std::string vlan_id) +{ + std::string res, cmds; + std::string link_add_cmd, link_set_master_cmd, link_up_cmd; + std::string bridge_add_cmd, bridge_untagged_add_cmd, bridge_del_vid_cmd; + std::string vxlan_dev_name; + + vxlan_dev_name = std::string("") + std::string(vxlanTunnelName) + "-" + + std::string(vlan_id); + + SWSS_LOG_INFO("Kernel tnl_name: %s vni_id: %s src_ip: %s dst_ip:%s vlan_id: %s", + vxlanTunnelName.c_str(), vni_id.c_str(), src_ip.c_str(), dst_ip.c_str(), + vlan_id.c_str()); + + // Case 1: Entry exist - Erase from cache & return + // Case 2: Enry does not exist - Create netDevice in Kernel + // Case 3: Entry exist but modified - Not taken care. Will address later + + if (m_in_reconcile) + { + auto it = m_vxlanNetDevices.find(vxlan_dev_name); + if (it != m_vxlanNetDevices.end()) + { + m_vxlanNetDevices.erase(it); + SWSS_LOG_INFO("Reconcile VxlanNetDevice %s reconciled. Pending %lu", + vxlan_dev_name.c_str(), m_vxlanNetDevices.size()); + return 0; + } + else + { + SWSS_LOG_INFO("Reconcile VxlanNetDevice %s does not exist. Pending %lu", + vxlan_dev_name.c_str(), m_vxlanNetDevices.size()); + } + } + else + { + SWSS_LOG_INFO("Creating VxlanNetDevice %s", vxlan_dev_name.c_str()); + } + + // ip link add type vxlan id local remote + // dstport 4789 + // ip link set master DOT1Q_BRIDGE_NAME + // bridge vlan add vid dev + // bridge vlan add vid untagged pvid dev + // ip link set up + + link_add_cmd = std::string("") + IP_CMD + " link add " + vxlan_dev_name + + " address " + gMacAddress.to_string() + " type vxlan id " + + std::string(vni_id) + " local " + src_ip + + ((dst_ip == "")? "":(" remote " + dst_ip)) + + " nolearning " + " dstport 4789 "; + + link_set_master_cmd = std::string("") + IP_CMD + " link set " + + vxlan_dev_name + " master Bridge "; + + link_up_cmd = std::string("") + IP_CMD + " link set " + vxlan_dev_name + " up "; + + bridge_add_cmd = std::string("") + BRIDGE_CMD + " vlan add vid " + + std::string(vlan_id) + " dev " + vxlan_dev_name; + + bridge_untagged_add_cmd = std::string("") + BRIDGE_CMD + " vlan add vid " + + std::string(vlan_id) + " untagged pvid dev " + vxlan_dev_name; + + bridge_del_vid_cmd = std::string("") + BRIDGE_CMD + " vlan del vid 1 dev " + + vxlan_dev_name; + + + cmds = std::string("") + BASH_CMD + " -c \"" + + link_add_cmd + " && " + + link_set_master_cmd + " && " + + bridge_add_cmd + " && " + + bridge_untagged_add_cmd + " && "; + + if ( vlan_id != "1") + { + cmds += bridge_del_vid_cmd + " && "; + } + + cmds += link_up_cmd + "\""; + + return swss::exec(cmds,res); +} + +int VxlanMgr::deleteVxlanNetdevice(std::string vxlan_dev_name) +{ + std::string res; + const std::string cmd = std::string("") + IP_CMD + " link del dev " + vxlan_dev_name; + return swss::exec(cmd, res); +} + +void VxlanMgr::getAllVxlanNetDevices() { std::string stdout; - const std::string cmd = std::string("") + IP_CMD + " link"; + const std::string cmd = std::string("") + IP_CMD + " link show type vxlan"; int ret = swss::exec(cmd, stdout); if (ret != 0) { @@ -600,22 +1046,181 @@ void VxlanMgr::clearAllVxlanDevices() auto lines = tokenize(stdout, '\n'); for (const std::string & line : lines) { + SWSS_LOG_INFO("line : %s\n",line.c_str()); if (!std::regex_search(line, match_result, device_name_pattern)) { continue; } - std::string res; - std::string device_name = match_result[1]; - VxlanInfo info; - if (device_name.find(VXLAN_NAME_PREFIX) == 0) + std::string vxlan_dev_name = match_result[1]; + m_vxlanNetDevices[vxlan_dev_name] = vxlan_dev_name; + } + return; +} + +void VxlanMgr::restoreVxlanNetDevices() +{ + /* Fetch the Src_ip from the vxlanAppTunnelTable */ + /* Currently, there is only 1 src vtep tunnel, hence picking the src_ip from that entry */ + Table vxlanAppTunnelTable = Table(this->m_app_db, APP_VXLAN_TUNNEL_TABLE_NAME); + vector appVxlanTunnelTableKeys; + vxlanAppTunnelTable.getKeys(appVxlanTunnelTableKeys); + std::string src_ip; + std::string dst_ip(""); + dst_ip = ""; + for (auto &k : appVxlanTunnelTableKeys) + { + std::vector temp; + if (vxlanAppTunnelTable.get(k, temp)) { - info.m_vxlan = device_name; - cmdDeleteVxlan(info, res); + for (auto fv: temp) + { + std::string field = fvField(fv); + std::string value = fvValue(fv); + SWSS_LOG_INFO("RESTORE Vxlan Tunnel Table key: %s field: %s value: %s", + k.c_str(), field.c_str(), value.c_str()); + if (field == "src_ip") + { + src_ip = value; + SWSS_LOG_INFO("RESTORE Vxlan Tunnel Table src_ip: %s", src_ip.c_str()); + } + } } - else if (device_name.find(VXLAN_IF_NAME_PREFIX) == 0) + else { - info.m_vxlanIf = device_name; - cmdDeleteVxlanIf(info, res); + SWSS_LOG_INFO("RESTORE VxLAN Tunnel Table Key(%s)", k.c_str()); } } + + Table vxlanAppTunnelMapTable = Table(this->m_app_db, APP_VXLAN_TUNNEL_MAP_TABLE_NAME); + std::vector::iterator it; + for (it = m_appVxlanTunnelMapKeysRecon.begin(); + it != m_appVxlanTunnelMapKeysRecon.end(); + it++) + { + std::string vlan, vlan_id, vni_id; + std::string vxlanTunnelMapName = *it; + std::vector temp; + if (vxlanAppTunnelMapTable.get(vxlanTunnelMapName, temp)) + { + for (auto fv: temp) + { + std::string field = fvField(fv); + std::string value = fvValue(fv); + SWSS_LOG_INFO("RESTORE Vxlan Tunnel MAP Table key: %s field: %s value: %s", + vxlanTunnelMapName.c_str(), field.c_str(), value.c_str()); + if (field == VLAN) + { + vlan = value; + } + else if (field == VNI) + { + vni_id = value; + } + } + } + else + { + SWSS_LOG_INFO("RESTORE VxLAN Tunnel Map Table Key(%s)", vxlanTunnelMapName.c_str()); + } + const auto vlan_prefix = std::string("Vlan"); + const auto prefix_len = vlan_prefix.length(); + vlan_id = vlan.substr(prefix_len); + + size_t found = vxlanTunnelMapName.find(delimiter); + const auto vxlanTunnelName = vxlanTunnelMapName.substr(0, found); + + int ret; + ret = createVxlanNetdevice(vxlanTunnelName, vni_id, src_ip, dst_ip, vlan_id); + if (ret != RET_SUCCESS) + { + SWSS_LOG_WARN("Vxlan Net Dev creation failure for %s VNI(%s) VLAN(%s)", + vxlanTunnelName.c_str(), vni_id.c_str(), vlan_id.c_str()); + } + + SWSS_LOG_INFO("RESTORE Created Kernel Net Device (%s-%s)", vxlanTunnelName.c_str(), vlan_id.c_str()); + } + + SWSS_LOG_INFO("RESTORE Delete Stale Kernel Net Devices"); + clearAllVxlanDevices(); + SWSS_LOG_INFO("RESTORE Recreate Kernel Cache"); + getAllVxlanNetDevices(); +} + +void VxlanMgr::clearAllVxlanDevices() +{ + for (auto it = m_vxlanNetDevices.begin(); it != m_vxlanNetDevices.end();) + { + SWSS_LOG_INFO("Deleting Stale NetDevice vxlandevname %s\n", (it->first).c_str()); + deleteVxlanNetdevice(it->first); + it = m_vxlanNetDevices.erase(it); + } +} + +void VxlanMgr::waitTillReadyToReconcile() +{ + for (;;) + { + WarmStart::WarmStartState state; + WarmStart::getWarmStartState("vlanmgrd", state); + + if ((WarmStart::REPLAYED == state) || + (WarmStart::RECONCILED == state)) + { + SWSS_LOG_INFO("Vlanmgrd Reconciled %d", (int) state); + return; + } + SWSS_LOG_INFO("Vlanmgrd NOT Reconciled %d", (int) state); + sleep(1); + } + return; +} + +void VxlanMgr::beginReconcile(bool warm) +{ + m_in_reconcile = true; + Table vxlanAppTunnelMapTable = Table(this->m_app_db, APP_VXLAN_TUNNEL_MAP_TABLE_NAME); + vxlanAppTunnelMapTable.getKeys(m_appVxlanTunnelMapKeysRecon); + for (auto &k : m_appVxlanTunnelMapKeysRecon) + { + SWSS_LOG_INFO("App Tunnel Map Key: %s", k.c_str()); + } + SWSS_LOG_INFO("Pending %lu entries for the Tunnel Map Table", m_appVxlanTunnelMapKeysRecon.size()); + return; +} + +void VxlanMgr::endReconcile(bool warm) +{ + /* Delete all stale entries from appDb */ + while (m_appVxlanTunnelMapKeysRecon.size()) + { + std::vector::iterator it = m_appVxlanTunnelMapKeysRecon.begin(); + if (it != m_appVxlanTunnelMapKeysRecon.end()) + { + SWSS_LOG_INFO("Reconcile Deleting Stale Entry vxlandevname %s\n", m_appVxlanTunnelMapKeysRecon[0].c_str()); + delAppDBTunnelMapTable(m_appVxlanTunnelMapKeysRecon[0]); + m_appVxlanTunnelMapKeysRecon.erase(it); + } + } + SWSS_LOG_INFO("End App Tunnel Map Table Reconcile"); + + /* Delete all the stale netDevices from the Kernel */ + clearAllVxlanDevices(); + + m_in_reconcile = false; +} + +bool VxlanMgr::isTunnelActive(std::string vxlanTunnelName) +{ + auto it = m_vxlanTunnelCache.find(vxlanTunnelName); + if (it == m_vxlanTunnelCache.end()) + { + return false; + } + + if (m_vxlanTunnelCache[vxlanTunnelName].m_sourceIp == "NULL") + { + return false; + } + + return true; } diff --git a/cfgmgr/vxlanmgr.h b/cfgmgr/vxlanmgr.h index d4a1fb0dc24d..0648e3d65ada 100644 --- a/cfgmgr/vxlanmgr.h +++ b/cfgmgr/vxlanmgr.h @@ -28,6 +28,27 @@ class VxlanMgr : public Orch std::string m_vxlanIf; std::string m_macAddress; } VxlanInfo; + + typedef struct TunCache + { + std::vector fvt; + std::string m_sourceIp; + uint32_t vlan_vni_refcnt; + } TunCache; + + typedef struct MapCache + { + std::string vxlan_dev_name; + std::string vlan; + std::string vni_id; + } MapCache; + + void waitTillReadyToReconcile(); + void beginReconcile(bool warm); + void endReconcile(bool warm); + void restoreVxlanNetDevices(); + bool isTunnelActive(std::string vxlanTunnelName); + ~VxlanMgr(); private: void doTask(Consumer &consumer); @@ -41,6 +62,16 @@ class VxlanMgr : public Orch bool doVxlanTunnelMapCreateTask(const KeyOpFieldsValuesTuple & t); bool doVxlanTunnelMapDeleteTask(const KeyOpFieldsValuesTuple & t); + bool doVxlanEvpnNvoCreateTask(const KeyOpFieldsValuesTuple & t); + bool doVxlanEvpnNvoDeleteTask(const KeyOpFieldsValuesTuple & t); + + void createAppDBTunnelMapTable(const KeyOpFieldsValuesTuple & t); + void delAppDBTunnelMapTable(std::string vxlanTunnelMapName); + int createVxlanNetdevice(std::string vxlanTunnelName, std::string vni_id, + std::string src_ip, std::string dst_ip, std::string vlan_id); + int deleteVxlanNetdevice(std::string vxlan_dev_name); + void getAllVxlanNetDevices(); + /* * Query the state of vrf by STATE_VRF_TABLE * Return @@ -49,6 +80,7 @@ class VxlanMgr : public Orch */ bool isVrfStateOk(const std::string & vrfName); bool isVxlanStateOk(const std::string & vxlanName); + bool isVlanStateOk(const std::string &vlanName); std::pair getVxlanRouterMacAddress(); bool createVxlan(const VxlanInfo & info); @@ -56,21 +88,32 @@ class VxlanMgr : public Orch void clearAllVxlanDevices(); - ProducerStateTable m_appVxlanTunnelTable,m_appVxlanTunnelMapTable; + ProducerStateTable m_appVxlanTunnelTable,m_appVxlanTunnelMapTable,m_appEvpnNvoTable; Table m_cfgVxlanTunnelTable,m_cfgVnetTable,m_stateVrfTable,m_stateVxlanTable, m_appSwitchTable; + Table m_stateVlanTable, m_stateTunnelVlanMapTable, m_stateVxlanTunnelTable; /* * Vxlan Tunnel Cache * Key: tunnel name * Value: Field Value pairs of vxlan tunnel */ - std::map > m_vxlanTunnelCache; + std::map m_vxlanTunnelCache; + std::map m_vxlanTunnelMapCache; + std::map m_vlanMapCache; + std::map m_vniMapCache; + std::map m_EvpnNvoCache; + /* * Vnet Cache * Key: Vnet name * Value: Vxlan information of this vnet */ std::map m_vnetCache; + + DBConnector *m_app_db; + bool m_in_reconcile; + std::vector m_appVxlanTunnelMapKeysRecon; + std::map m_vxlanNetDevices; }; } diff --git a/cfgmgr/vxlanmgrd.cpp b/cfgmgr/vxlanmgrd.cpp index 8e86cfbe4976..19b827b17a9b 100644 --- a/cfgmgr/vxlanmgrd.cpp +++ b/cfgmgr/vxlanmgrd.cpp @@ -13,6 +13,7 @@ #include "producerstatetable.h" #include "vxlanmgr.h" #include "shellcmd.h" +#include "warm_restart.h" using namespace std; using namespace swss; @@ -35,6 +36,7 @@ ofstream gRecordOfs; string gRecordFile; /* Global database mutex */ mutex gDbMutex; +MacAddress gMacAddress; int main(int argc, char **argv) { @@ -49,10 +51,18 @@ int main(int argc, char **argv) DBConnector appDb("APPL_DB", 0); DBConnector stateDb("STATE_DB", 0); + WarmStart::initialize("vxlanmgrd", "swss"); + WarmStart::checkWarmStart("vxlanmgrd", "swss"); + if (WarmStart::isWarmStart()) + { + WarmStart::setWarmStartState("vxlanmgrd", WarmStart::INITIALIZED); + } + vector cfg_vnet_tables = { CFG_VNET_TABLE_NAME, CFG_VXLAN_TUNNEL_TABLE_NAME, CFG_VXLAN_TUNNEL_MAP_TABLE_NAME, + CFG_VXLAN_EVPN_NVO_TABLE_NAME, }; VxlanMgr vxlanmgr(&cfgDb, &appDb, &stateDb, cfg_vnet_tables); @@ -65,6 +75,48 @@ int main(int argc, char **argv) s.addSelectables(o->getSelectables()); } + /* + * swss service starts after interfaces-config.service which will have + * switch_mac set. + * Dynamic switch_mac update is not supported for now. + */ + Table table(&cfgDb, "DEVICE_METADATA"); + std::vector ovalues; + table.get("localhost", ovalues); + auto it = std::find_if( ovalues.begin(), ovalues.end(), [](const FieldValueTuple& t){ return t.first == "mac";} ); + if ( it == ovalues.end() ) { + throw runtime_error("couldn't find MAC address of the device from config DB"); + } + gMacAddress = MacAddress(it->second); + + auto in_recon = true; + vxlanmgr.beginReconcile(true); + + if (WarmStart::isWarmStart()) + { + WarmStart::WarmStartState state; + + vxlanmgr.waitTillReadyToReconcile(); + vxlanmgr.restoreVxlanNetDevices(); + WarmStart::setWarmStartState("vxlanmgrd", WarmStart::REPLAYED); + uint16_t wait_secs = 0; + string val = ""; + Table wb_tbl = Table(&stateDb, STATE_WARM_RESTART_TABLE_NAME); + wb_tbl.hget("orchagent", "restore_count", val); + if ((val != "") or (val != "0")) + { + WarmStart::getWarmStartState("orchagent",state); + while (state != WarmStart::RECONCILED) + { + SWSS_LOG_NOTICE("Waiting Until Orchagent is reconciled. Current %s. Waited %u secs", + val.c_str(), wait_secs); + sleep(1); + wait_secs++; + WarmStart::getWarmStartState("orchagent",state); + } + } + } + SWSS_LOG_NOTICE("starting main loop"); while (true) { @@ -79,6 +131,15 @@ int main(int argc, char **argv) } if (ret == Select::TIMEOUT) { + if (true == in_recon) + { + in_recon = false; + vxlanmgr.endReconcile(false); + if (WarmStart::isWarmStart()) + { + WarmStart::setWarmStartState("vxlanmgrd", WarmStart::RECONCILED); + } + } vxlanmgr.doTask(); continue; } diff --git a/orchagent/portsorch.cpp b/orchagent/portsorch.cpp index 85133130537f..25a64b553bd7 100755 --- a/orchagent/portsorch.cpp +++ b/orchagent/portsorch.cpp @@ -1853,7 +1853,7 @@ bool PortsOrch::initPort(const string &alias, const int index, const set &l /* Determine if the port has already been initialized before */ if (m_portList.find(alias) != m_portList.end() && m_portList[alias].m_port_id == id) { - SWSS_LOG_INFO("Port has already been initialized before alias:%s", alias.c_str()); + SWSS_LOG_DEBUG("Port has already been initialized before alias:%s", alias.c_str()); } else { diff --git a/orchagent/vxlanorch.cpp b/orchagent/vxlanorch.cpp index d68cac724168..f4d24f6a4c0b 100644 --- a/orchagent/vxlanorch.cpp +++ b/orchagent/vxlanorch.cpp @@ -845,7 +845,7 @@ bool VxlanTunnel::deleteTunnelHw(uint8_t mapper_list, tunnel_map_use_t map_src, remove_tunnel_termination(ids_.tunnel_term_id); } - remove_tunnel_termination(ids_.tunnel_id); + remove_tunnel(ids_.tunnel_id); deleteMapperHw(mapper_list, map_src); }