From 3952aabdaf486786f1b1908dca6e7028097a7060 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 15 Oct 2021 17:19:08 +0100 Subject: [PATCH 01/29] [cbf] Added initial CBF support (#2) * Added CBF NHG support to NhgOrch * Added NhgMapOrch to handle DSCP_TO_FC and EXP_TO_FC tables * Added UT for the new NhgOrch function and NhgMapOrch Signed-off-by: Alexandru Banu --- doc/Configuration.md | 2 + doc/swss-schema.md | 57 ++ orchagent/Makefile.am | 5 +- orchagent/cbfnhghandler.cpp | 788 +++++++++++++++ orchagent/cbfnhghandler.h | 96 ++ orchagent/mplsrouteorch.cpp | 8 +- orchagent/neighorch.cpp | 4 +- orchagent/nhgbase.cpp | 45 + orchagent/nhgbase.h | 465 +++++++++ orchagent/{nhgorch.cpp => nhghandler.cpp} | 368 ++----- orchagent/nhghandler.h | 116 +++ orchagent/nhgmaporch.cpp | 366 +++++++ orchagent/nhgmaporch.h | 61 ++ orchagent/nhgorch.h | 309 ++---- orchagent/orchdaemon.cpp | 13 +- orchagent/orchdaemon.h | 1 + orchagent/qosorch.cpp | 192 +++- orchagent/qosorch.h | 18 + orchagent/routeorch.cpp | 12 +- tests/mock_tests/Makefile.am | 5 +- tests/test_nhg.py | 1086 +++++++++++++++++---- tests/test_qos_map.py | 155 ++- 22 files changed, 3497 insertions(+), 675 deletions(-) create mode 100644 orchagent/cbfnhghandler.cpp create mode 100644 orchagent/cbfnhghandler.h create mode 100644 orchagent/nhgbase.cpp create mode 100644 orchagent/nhgbase.h rename orchagent/{nhgorch.cpp => nhghandler.cpp} (71%) create mode 100644 orchagent/nhghandler.h create mode 100644 orchagent/nhgmaporch.cpp create mode 100644 orchagent/nhgmaporch.h diff --git a/doc/Configuration.md b/doc/Configuration.md index 0c91e16db5..039e749d37 100644 --- a/doc/Configuration.md +++ b/doc/Configuration.md @@ -1129,6 +1129,8 @@ name as object key and member list as attribute. "pfc_enable": "3,4", "pfc_to_queue_map": "AZURE", "dscp_to_tc_map": "AZURE", + "dscp_to_fc_map": "AZURE", + "exp_to_fc_map": "AZURE", "scheduler": "scheduler.port" } } diff --git a/doc/swss-schema.md b/doc/swss-schema.md index de89c02f9e..1fe9ef9b2c 100644 --- a/doc/swss-schema.md +++ b/doc/swss-schema.md @@ -194,6 +194,28 @@ and reflects the LAG ports into the redis under: `LAG_TABLE::port` mpls_nh = STRING ; Comma-separated list of MPLS NH info. weight = weight_list ; List of weights. +--------------------------------------------- +### CLASS_BASED_NEXT_HOP_GROUP_TABLE + ;Stores a list of groups of one or more next hop groups used for class based forwarding + ;Status: Mandatory + key = CLASS_BASED_NEXT_HOP_GROUP_TABLE:string ; arbitrary index for the next hop group + members = NEXT_HOP_GROUP_TABLE.key ; one or more separated by "," + selection_map = FC_TO_NHG_INDEX_MAP_TABLE.key ; the NHG map to use for this CBF NHG + +--------------------------------------------- +### FC_TO_NHG_INDEX_MAP_TABLE + ; FC to Next hop group index map + key = "FC_TO_NHG_INDEX_MAP_TABLE:"name + fc_num = 1*DIGIT ;value + nh_index = 1*DIGIT; index of NH inside NH group + + Example: + 127.0.0.1:6379> hgetall "FC_TO_NHG_INDEX_MAP_TABLE:AZURE" + 1) "0" ;fc_num + 2) "0" ;nhg_index + 3) "1" + 4) "0" + --------------------------------------------- ### NEIGH_TABLE ; Stores the neighbors or next hop IP address and output port or @@ -321,6 +343,41 @@ and reflects the LAG ports into the redis under: `LAG_TABLE::port` 9) "4" 10) "8" +### DSCP_TO_FC_TABLE_NAME + ; dscp to FC map + ;SAI mapping - qos_map object with SAI_QOS_MAP_ATTR_TYPE == sai_qos_map_type_t::SAI_QOS_MAP_TYPE_DSCP_TO_FORWARDING_CLASS + key = "DSCP_TO_FC_MAP_TABLE:"name + ;field value + dscp_value = 1*DIGIT + fc_value = 1*DIGIT + + Example: + 127.0.0.1:6379> hgetall "DSCP_TO_FC_MAP_TABLE:AZURE" + 1) "0" ;dscp + 2) "1" ;fc + 3) "1" + 4) "1" + 5) "2" + 6) "3" + 7) +--------------------------------------------- +### EXP_TO_FC_MAP_TABLE + ; dscp to FC map + ;SAI mapping - qos_map object with SAI_QOS_MAP_ATTR_TYPE == sai_qos_map_type_t::SAI_QOS_MAP_TYPE_MPLS_EXP_TO_FORWARDING_CLASS + key = "EXP_TO_FC_MAP_TABLE:"name + ;field value + mpls_exp_value = 1*DIGIT + fc_value = 1*DIGIT + + Example: + 127.0.0.1:6379> hgetall "EXP_TO_FC_MAP_TABLE:AZURE" + 1) "0" ;mpls_exp + 2) "1" ;fc + 3) "1" + 4) "1" + 5) "2" + 6) "3" + --------------------------------------------- ### SCHEDULER_TABLE ; Scheduler table diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index cf789b027f..07007bbe52 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -40,7 +40,10 @@ orchagent_SOURCES = \ orchdaemon.cpp \ orch.cpp \ notifications.cpp \ - nhgorch.cpp \ + nhgbase.cpp \ + nhghandler.cpp \ + cbfnhghandler.cpp \ + nhgmaporch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ neighorch.cpp \ diff --git a/orchagent/cbfnhghandler.cpp b/orchagent/cbfnhghandler.cpp new file mode 100644 index 0000000000..621895a36d --- /dev/null +++ b/orchagent/cbfnhghandler.cpp @@ -0,0 +1,788 @@ +#include "cbfnhghandler.h" +#include "crmorch.h" +#include "bulker.h" +#include "tokenize.h" +#include "nhgorch.h" +#include "nhgmaporch.h" +#include "routeorch.h" + +extern sai_object_id_t gSwitchId; + +extern NhgOrch *gNhgOrch; +extern CrmOrch *gCrmOrch; +extern NhgMapOrch *gNhgMapOrch; +extern RouteOrch *gRouteOrch; + +extern sai_next_hop_group_api_t* sai_next_hop_group_api; + +extern size_t gMaxBulkSize; + +/* + * Purpose: Perform the operations requested by APPL_DB users. + * + * Description: Iterate over the untreated operations list and resolve them. + * The operations supported are SET and DEL. If an operation + * could not be resolved, it will either remain in the list, or be + * removed, depending on the case. + * + * Params: IN consumer - The cosumer object. + * + * Returns: Nothing. + */ +void CbfNhgHandler::doTask(Consumer& consumer) +{ + SWSS_LOG_ENTER(); + + auto it = consumer.m_toSync.begin(); + + while (it != consumer.m_toSync.end()) + { + swss::KeyOpFieldsValuesTuple t = it->second; + + string index = kfvKey(t); + string op = kfvOp(t); + + bool success; + const auto &cbf_nhg_it = m_syncedNextHopGroups.find(index); + + if (op == SET_COMMAND) + { + string members; + string selection_map; + + /* + * Get CBF group's members and selection map. + */ + for (const auto &i : kfvFieldsValues(t)) + { + if (fvField(i) == "members") + { + members = fvValue(i); + } + else if (fvField(i) == "selection_map") + { + selection_map = fvValue(i); + } + } + + /* + * Validate the data. + */ + auto p = getMembers(members); + + if (!p.first) + { + SWSS_LOG_ERROR("CBF next hop group %s data is invalid.", + index.c_str()); + it = consumer.m_toSync.erase(it); + continue; + } + + /* + * If the CBF group does not exist, create it. + */ + if (cbf_nhg_it == m_syncedNextHopGroups.end()) + { + /* + * If we reached the NHG limit, postpone the creation. + */ + if (gRouteOrch->getNhgCount() + NhgBase::getSyncedCount() >= gRouteOrch->getMaxNhgCount()) + { + SWSS_LOG_WARN("Reached next hop group limit. Postponing creation."); + success = false; + } + else + { + auto cbf_nhg = CbfNhg(index, p.second, selection_map); + success = cbf_nhg.sync(); + + if (success) + { + /* + * If the CBF NHG contains temporary NHGs as members, + * we have to keep checking for updates. + */ + if (cbf_nhg.hasTemps()) + { + success = false; + } + + m_syncedNextHopGroups.emplace(index, NhgEntry(move(cbf_nhg))); + } + } + } + /* + * If the CBF group exists, update it. + */ + else + { + success = cbf_nhg_it->second.nhg.update(p.second, selection_map); + + /* + * If the CBF NHG has temporary NHGs synced, we need to keep + * checking this group in case they are promoted. + */ + if (cbf_nhg_it->second.nhg.hasTemps()) + { + success = false; + } + } + } + else if (op == DEL_COMMAND) + { + /* + * If there is a pending SET after this DEL operation, skip the + * DEL operation to perform the update instead. Otherwise, in the + * scenario where the DEL operation may be blocked by the ref + * counter, we'd end up deleting the object after the SET operation + * is performed, which would not reflect the desired state of the + * object. + */ + if (consumer.m_toSync.count(it->first) > 1) + { + success = true; + } + /* If the group doesn't exist, do nothing. */ + else if (cbf_nhg_it == m_syncedNextHopGroups.end()) + { + SWSS_LOG_WARN("Deleting inexistent CBF NHG %s", index.c_str()); + /* + * Mark it as a success to remove the task from the consumer. + */ + success = true; + } + /* If the group does exist but is still referenced, skip.*/ + else if (cbf_nhg_it->second.ref_count > 0) + { + SWSS_LOG_WARN("Skipping removal of CBF next hop group %s which" + " is still referenced", index.c_str()); + success = false; + } + /* Otherwise, delete it. */ + else + { + success = cbf_nhg_it->second.nhg.remove(); + + if (success) + { + m_syncedNextHopGroups.erase(cbf_nhg_it); + } + } + } + else + { + SWSS_LOG_WARN("Unknown operation type %s", op.c_str()); + /* Mark the operation as successful to consume it. */ + success = true; + } + + /* Depending on the operation success, consume it or skip it. */ + if (success) + { + it = consumer.m_toSync.erase(it); + } + else + { + ++it; + } + } +} + +/* + * Purpose: Validate the CBF members. + * + * Params: IN members - The members string to validate. + * + * Returns: Pair where: + * - the first element is a bool, representing if the members are + * valid or not + * - the second element is a vector of members + */ +pair> CbfNhgHandler::getMembers(const string &members) +{ + SWSS_LOG_ENTER(); + + auto members_vec = swss::tokenize(members, ','); + set members_set(members_vec.begin(), members_vec.end()); + bool success = true; + + /* + * Verify that the members list is not empty + */ + if (members_set.empty()) + { + SWSS_LOG_ERROR("CBF next hop group members list is empty."); + success = false; + } + /* + * Verify that the members are unique. + */ + else if (members_set.size() != members_vec.size()) + { + SWSS_LOG_ERROR("CBF next hop group members are not unique."); + success = false; + } + + return {success, members_vec}; +} + +/* + * Purpose: Constructor. + * + * Params: IN index - The index of the CBF NHG. + * IN members - The members of the CBF NHG. + * IN selection_map - The selection map index of the CBF NHG. + * + * Returns: Nothing. + */ +CbfNhg::CbfNhg(const string &index, + const vector &members, + const string &selection_map) : + NhgCommon(index), + m_selection_map(selection_map) +{ + SWSS_LOG_ENTER(); + + uint8_t idx = 0; + for (const auto &member : members) + { + m_members.emplace(member, CbfNhgMember(member, idx++)); + } +} + +/* + * Purpose: Move constructor. + * + * Params: IN cbf_nhg - The temporary object to construct from. + * + * Returns: Nothing. + */ +CbfNhg::CbfNhg(CbfNhg &&cbf_nhg) : + NhgCommon(move(cbf_nhg)), + m_selection_map(move(cbf_nhg.m_selection_map)), + m_temp_nhgs(move(cbf_nhg.m_temp_nhgs)) +{ + SWSS_LOG_ENTER(); +} + +/* + * Purpose: Sync the CBF NHG over SAI, getting a SAI ID. + * + * Params: None. + * + * Returns: true, if the operation was successful, + * false, otherwise. + */ +bool CbfNhg::sync() +{ + SWSS_LOG_ENTER(); + + /* If the group is already synced, exit. */ + if (isSynced()) + { + return true; + } + + /* Create the CBF next hop group over SAI. */ + sai_attribute_t nhg_attr; + vector nhg_attrs; + + nhg_attr.id = SAI_NEXT_HOP_GROUP_ATTR_TYPE; + nhg_attr.value.u32 = SAI_NEXT_HOP_GROUP_TYPE_CLASS_BASED; + nhg_attrs.push_back(move(nhg_attr)); + + /* Add the number of members. */ + nhg_attr.id = SAI_NEXT_HOP_GROUP_ATTR_CONFIGURED_SIZE; + assert(m_members.size() <= UINT32_MAX); + nhg_attr.value.u32 = static_cast(m_members.size()); + nhg_attrs.push_back(move(nhg_attr)); + + /* Add the selection map to the attributes. */ + nhg_attr.id = SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP; + nhg_attr.value.oid = gNhgMapOrch->getMapId(m_selection_map); + + if (nhg_attr.value.oid == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_WARN("FC to NHG map index %s does not exist", m_selection_map.c_str()); + return false; + } + + nhg_attrs.push_back(move(nhg_attr)); + + auto status = sai_next_hop_group_api->create_next_hop_group( + &m_id, + gSwitchId, + static_cast(nhg_attrs.size()), + nhg_attrs.data()); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create CBF next hop group %s, rv %d", + m_key.c_str(), + status); + task_process_status handle_status = gNhgOrch->handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status); + if (handle_status != task_success) + { + return gNhgOrch->parseHandleSaiStatusFailure(handle_status); + } + } + + /* Increase the reference counter for the selection map. */ + gNhgMapOrch->incRefCount(m_selection_map); + + /* + * Increment the amount of programmed next hop groups. */ + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); + incSyncedCount(); + + /* Sync the group members. */ + set members; + + for (const auto &member : m_members) + { + members.insert(member.first); + } + + if (!syncMembers(members)) + { + SWSS_LOG_ERROR("Failed to sync CBF next hop group %s", m_key.c_str()); + return false; + } + + return true; +} + +/* + * Purpose: Remove the CBF NHG over SAI, removing the SAI ID. + * + * Params: None. + * + * Returns: true, if the operation was successful, + * false, otherwise. + */ +bool CbfNhg::remove() +{ + SWSS_LOG_ENTER(); + + bool is_synced = isSynced(); + + bool success = NhgCommon::remove(); + + if (success && is_synced) + { + gNhgMapOrch->decRefCount(m_selection_map); + } + + return success; +} + +/* + * Purpose: Check if the CBF NHG has the same members and in the same order as + * the ones given. + * + * Params: IN members - The members to compare with. + * + * Returns: true, if the current members are the same with the given one, + * false, otherwise. + */ +bool CbfNhg::hasSameMembers(const vector &members) const +{ + SWSS_LOG_ENTER(); + + /* The size should be the same. */ + if (m_members.size() != members.size()) + { + return false; + } + + /* + * Check that the members are the same and the index is preserved. + */ + uint8_t index = 0; + + for (const auto &member : members) + { + auto mbr_it = m_members.find(member); + + if (mbr_it == m_members.end()) + { + return false; + } + + if (mbr_it->second.getIndex() != index++) + { + return false; + } + } + + return true; +} + +/* + * Purpose: Update a CBF next hop group. + * + * Params: IN members - The new members. + * IN selection_map - The new selection map. + * + * Returns: true, if the update was successful, + * false, otherwise. + */ +bool CbfNhg::update(const vector &members, const string &selection_map) +{ + SWSS_LOG_ENTER(); + + /* + * Check if we're just checking if the temporary NHG members were updated. + * This would happen only if the given members are the same with the + * existing ones and in the same order. In this scenario, we'll update + * just the NEXT_HOP attribute of the temporary members if necessary. + */ + if (!m_temp_nhgs.empty() && hasSameMembers(members)) + { + /* Iterate over the temporary NHGs and check if they were updated. */ + for (auto member = m_temp_nhgs.begin(); member != m_temp_nhgs.end();) + { + const auto &nhg = gNhgOrch->getNhg(member->first); + + /* + * If the NHG ID has changed since storing the first occurence, + * we have to update the CBF NHG member's attribute. + */ + if (nhg.getId() != member->second) + { + if (!m_members.at(member->first).updateNhAttr()) + { + SWSS_LOG_ERROR("Failed to update temporary next hop group " + "member %s of CBF next hop group %s", + member->first.c_str(), m_key.c_str()); + return false; + } + + /* If the NHG was promoted, remove it from the temporary NHG map. */ + if (!nhg.isTemp()) + { + member = m_temp_nhgs.erase(member); + } + /* + * If the NHG was just updated, update its current NHG ID value + * in the map. + */ + else + { + member->second = nhg.getId(); + ++member; + } + } + else + { + ++member; + } + } + } + /* If the members are different, update the whole members list. */ + else + { + /* + * Because the INDEX attribute is CREATE_ONLY, we have to remove all + * the existing members and sync the new ones back as the order of the + * members can change due to some of them being removed, which would + * invalidate all the other members following them, or inserting a new + * one somewhere in the front of the existing members which would also + * invalidate them. + */ + set members_set; + + for (const auto &member : m_members) + { + members_set.insert(member.first); + } + + /* Remove the existing members. */ + if (!removeMembers(members_set)) + { + SWSS_LOG_ERROR("Failed to remove members of CBF next hop group %s", + m_key.c_str()); + return false; + } + m_members.clear(); + m_temp_nhgs.clear(); + + /* Add the new members. */ + uint8_t index = 0; + for (const auto &member : members) + { + m_members.emplace(member, CbfNhgMember(member, index++)); + } + + /* Sync the new members. */ + if (!syncMembers({members.begin(), members.end()})) + { + SWSS_LOG_ERROR("Failed to sync members of CBF next hop group %s", + m_key.c_str()); + return false; + } + } + + /* Update the group map. */ + if (m_selection_map != selection_map) + { + sai_attribute_t nhg_attr; + nhg_attr.id = SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP; + nhg_attr.value.oid = gNhgMapOrch->getMapId(selection_map); + + if (nhg_attr.value.oid == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_WARN("NHG map %s does not exist", selection_map.c_str()); + return false; + } + + auto status = sai_next_hop_group_api->set_next_hop_group_attribute( m_id, &nhg_attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to update CBF next hop group %s, rv %d", + m_key.c_str(), + status); + return false; + } + + /* Update the selection map and update the previous and new map ref count. */ + gNhgMapOrch->decRefCount(m_selection_map); + m_selection_map = selection_map; + gNhgMapOrch->incRefCount(m_selection_map); + } + + return true; +} + +/* + * Purpose: Sync the given CBF group members. + * + * Params: IN members - The members to sync. + * + * Returns: true, if the operation was successful, + * false, otherwise. + */ +bool CbfNhg::syncMembers(const set &members) +{ + SWSS_LOG_ENTER(); + + /* The group should be synced at this point. */ + if (!isSynced()) + { + SWSS_LOG_ERROR("Trying to sync members of CBF next hop group %s which is not synced", + m_key.c_str()); + throw logic_error("Syncing members of unsynced CBF next hop group"); + } + + /* + * Sync all the given members. If a NHG does not exist, is not yet synced + * or is temporary, stop immediately. + */ + ObjectBulker bulker(sai_next_hop_group_api, + gSwitchId, + gMaxBulkSize); + unordered_map nhgm_ids; + + for (const auto &key : members) + { + auto &nhgm = m_members.at(key); + + /* A next hop group member can't be already synced if it was passed into the syncMembers() method. */ + if (nhgm.isSynced()) + { + SWSS_LOG_ERROR("Trying to sync already synced CBF NHG member %s in group %s", + nhgm.to_string().c_str(), to_string().c_str()); + throw std::logic_error("Syncing already synced NHG member"); + } + + /* + * Check if the group exists in NhgOrch. + */ + if (!gNhgOrch->nhgHandler.hasNhg(key)) + { + SWSS_LOG_ERROR("Next hop group %s in CBF next hop group %s does " + "not exist", key.c_str(), m_key.c_str()); + return false; + } + + const auto &nhg = gNhgOrch->nhgHandler.getNhg(key); + + /* + * Check if the group is synced. + */ + if (!nhg.isSynced()) + { + SWSS_LOG_ERROR("Next hop group %s in CBF next hop group %s is not" + " synced", + key.c_str(), m_key.c_str()); + return false; + } + + /* Create the SAI attributes for syncing the NHG as a member. */ + auto attrs = createNhgmAttrs(nhgm); + + bulker.create_entry(&nhgm_ids[key], + (uint32_t)attrs.size(), + attrs.data()); + } + + /* Flush the bulker to perform the sync. */ + bulker.flush(); + + /* Iterate over the synced members and set their SAI ID. */ + bool success = true; + + for (const auto &member : nhgm_ids) + { + if (member.second == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_ERROR("Failed to create CBF next hop group %s member %s", + m_key.c_str(), member.first.c_str()); + success = false; + } + else + { + m_members.at(member.first).sync(member.second); + + const auto &nhg = gNhgOrch->getNhg(member.first); + /* + * If the member is temporary, store it in order to check if it is + * promoted at some point. + */ + if (nhg.isTemp()) + { + m_temp_nhgs.emplace(member.first, nhg.getId()); + } + } + } + + return success; +} + +/* + * Purpose: Create a vector with the SAI attributes for syncing a next hop + * group member over SAI. The caller is reponsible of filling in the + * index attribute. + * + * Params: IN nhgm - The next hop group member to sync. + * + * Returns: The vector containing the SAI attributes. + */ +vector CbfNhg::createNhgmAttrs(const CbfNhgMember &nhgm) const +{ + SWSS_LOG_ENTER(); + + if (!isSynced() || (nhgm.getNhgId() == SAI_NULL_OBJECT_ID)) + { + SWSS_LOG_ERROR("CBF next hop group %s or next hop group %s are not synced", + to_string().c_str(), nhgm.to_string().c_str()); + throw logic_error("CBF next hop group member attributes data is insufficient"); + } + + vector attrs; + sai_attribute_t attr; + + /* Fill in the group ID. */ + attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID; + attr.value.oid = m_id; + attrs.push_back(attr); + + /* Fill in the next hop ID. */ + attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID; + attr.value.oid = nhgm.getNhgId(); + attrs.push_back(attr); + + /* Fill in the index. */ + attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX; + attr.value.oid = nhgm.getIndex(); + attrs.push_back(attr); + + return attrs; +} + +/* + * Purpose: Sync the member, setting its SAI ID and incrementing the necessary + * ref counters. + * + * Params: IN gm_id - The SAI ID to set. + * + * Returns: Nothing. + */ +void CbfNhgMember::sync(sai_object_id_t gm_id) +{ + SWSS_LOG_ENTER(); + + NhgMember::sync(gm_id); + gNhgOrch->nhgHandler.incNhgRefCount(m_key); +} + +/* + * Purpose: Update the next hop attribute of the member. + * + * Params: None. + * + * Returns: true, if the operation was successful, + * false, otherwise. + */ +bool CbfNhgMember::updateNhAttr() +{ + SWSS_LOG_ENTER(); + + if (!isSynced()) + { + SWSS_LOG_ERROR("Trying to update next hop attribute of CBF NHG member %s that is not synced", + to_string().c_str()); + throw logic_error("Trying to update attribute of unsynced object."); + } + + /* + * Fill in the attribute. + */ + sai_attribute_t attr; + attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID; + attr.value.oid = getNhgId(); + + /* + * Set the attribute over SAI. + */ + auto status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_id, &attr); + + return status == SAI_STATUS_SUCCESS; +} + +/* + * Purpose: Remove the member, reseting its SAI ID and decrementing the NHG ref + * counter. + * + * Params: None. + * + * Returns: Nothing. + */ +void CbfNhgMember::remove() +{ + SWSS_LOG_ENTER(); + + NhgMember::remove(); + gNhgOrch->nhgHandler.decNhgRefCount(m_key); +} + +/* + * Purpose: Get the NHG ID of this member. + * + * Params: None. + * + * Returns: The SAI ID of the NHG it references or SAI_NULL_OBJECT_ID if it + * doesn't exist. + */ +sai_object_id_t CbfNhgMember::getNhgId() const +{ + SWSS_LOG_ENTER(); + + if (!gNhgOrch->hasNhg(m_key)) + { + return SAI_NULL_OBJECT_ID; + } + + return gNhgOrch->getNhg(m_key).getId(); +} diff --git a/orchagent/cbfnhghandler.h b/orchagent/cbfnhghandler.h new file mode 100644 index 0000000000..af2dbb30af --- /dev/null +++ b/orchagent/cbfnhghandler.h @@ -0,0 +1,96 @@ +#pragma once + +#include "nhgbase.h" +#include "nhghandler.h" + +using namespace std; + +class CbfNhgMember : public NhgMember +{ +public: + CbfNhgMember(const string &key, uint8_t index) : + NhgMember(key), m_index(index) { SWSS_LOG_ENTER(); } + + /* Sync the member, setting its SAI ID and incrementing the necessary ref counters. */ + void sync(sai_object_id_t gm_id) override; + + /* Remove the member, reseting its SAI ID and decrementing the necessary ref counters. */ + void remove() override; + + /* Get the NHG ID of this member. */ + sai_object_id_t getNhgId() const; + + /* Update the NEXT_HOP attribute of this member. */ + bool updateNhAttr(); + + /* Get the index of this group member. */ + uint8_t getIndex() const { return m_index; } + + /* Get a string representation of this member. */ + string to_string() const override { return m_key; } + +private: + /* The index of this member in the group's member list. */ + uint8_t m_index; +}; + +class CbfNhg : public NhgCommon +{ +public: + /* Constructors. */ + CbfNhg(const string &index, + const vector &members, + const string &selection_map); + CbfNhg(CbfNhg &&cbf_nhg); + + /* Destructor. */ + ~CbfNhg() { SWSS_LOG_ENTER(); remove(); } + + /* Create the CBF group over SAI. */ + bool sync() override; + + /* Remove the CBF group over SAI. */ + bool remove() override; + + /* CBF groups can never be temporary. */ + inline bool isTemp() const override { SWSS_LOG_ENTER(); return false; } + + /* Check if the CBF next hop group contains synced temporary NHGs. */ + inline bool hasTemps() const + { SWSS_LOG_ENTER(); return !m_temp_nhgs.empty(); } + + /* CBF groups do not have a NextHopGroupkey. */ + inline NextHopGroupKey getNhgKey() const override { return {}; } + + /* Update the CBF group, including the SAI programming. */ + bool update(const vector &members, const string &selection_map); + + string to_string() const override { return m_key; } + +private: + string m_selection_map; + + /* + * Map of synced temporary NHGs contained in this next hop group along with + * the NHG ID at the time of sync. + */ + unordered_map m_temp_nhgs; + + /* Sync the given members over SAI. */ + bool syncMembers(const set &members) override; + + /* Get the SAI attributes for creating the members over SAI. */ + vector + createNhgmAttrs(const CbfNhgMember &nhg) const override; + /* Check if the CBF NHG has the same members and in the same order as the ones given. */ + bool hasSameMembers(const vector &members) const; +}; + +class CbfNhgHandler : public NhgHandlerCommon +{ +public: + void doTask(Consumer &consumer); + +private: + static pair> getMembers(const string &members); +}; diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 5aebf70cbe..86b7eea75e 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -253,8 +253,8 @@ void RouteOrch::doLabelTask(Consumer& consumer) { try { - const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); - ctx.nhg = nh_group.getKey(); + const auto &nh_group = gNhgOrch->getNhg(nhg_index); + ctx.nhg = nh_group.getNhgKey(); ctx.using_temp_nhg = nh_group.isTemp(); } catch (const std::out_of_range& e) @@ -308,7 +308,7 @@ void RouteOrch::doLabelTask(Consumer& consumer) // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (gNhgOrch->getNhgCount() >= gNhgOrch->getMaxNhgCount() && + if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount && gLabelRouteBulker.removing_entries_count() > 0) { break; @@ -478,7 +478,7 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey { try { - const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); + const auto &nhg = gNhgOrch->getNhg(ctx.nhg_index); next_hop_id = nhg.getId(); } catch(const std::out_of_range& e) diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index c1eba4c0e3..44c10d818a 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -334,7 +334,7 @@ bool NeighOrch::setNextHopFlag(const NextHopKey &nexthop, const uint32_t nh_flag { case NHFLAGS_IFDOWN: rc = gRouteOrch->invalidnexthopinNextHopGroup(nexthop, count); - rc &= gNhgOrch->invalidateNextHop(nexthop); + rc &= gNhgOrch->nhgHandler.invalidateNextHop(nexthop); break; default: assert(0); @@ -364,7 +364,7 @@ bool NeighOrch::clearNextHopFlag(const NextHopKey &nexthop, const uint32_t nh_fl { case NHFLAGS_IFDOWN: rc = gRouteOrch->validnexthopinNextHopGroup(nexthop, count); - rc &= gNhgOrch->validateNextHop(nexthop); + rc &= gNhgOrch->nhgHandler.validateNextHop(nexthop); break; default: assert(0); diff --git a/orchagent/nhgbase.cpp b/orchagent/nhgbase.cpp new file mode 100644 index 0000000000..390d38ccfb --- /dev/null +++ b/orchagent/nhgbase.cpp @@ -0,0 +1,45 @@ +#include "nhgbase.h" +#include "vector" +#include "rediscommand.h" + +extern sai_object_id_t gSwitchId; + +unsigned NhgBase::m_syncedCount = 0; + +/* + * Purpose: Destructor. + * + * Params: None. + * + * Returns: Nothing. + */ +NhgBase::~NhgBase() +{ + SWSS_LOG_ENTER(); + + if (isSynced()) + { + SWSS_LOG_ERROR("Destroying next hop group with SAI ID %lu which is still synced.", m_id); + assert(false); + } +} + +/* + * Purpose: Decrease the count of synced next hop group objects. + * + * Params: None. + * + * Returns: Nothing. + */ +void NhgBase::decSyncedCount() +{ + SWSS_LOG_ENTER(); + + if (m_syncedCount == 0) + { + SWSS_LOG_ERROR("Decreasing next hop groups count while already 0"); + throw logic_error("Decreasing next hop groups count while already 0"); + } + + --m_syncedCount; +} diff --git a/orchagent/nhgbase.h b/orchagent/nhgbase.h new file mode 100644 index 0000000000..f718b76f1a --- /dev/null +++ b/orchagent/nhgbase.h @@ -0,0 +1,465 @@ +#pragma once + +#include "string" +#include "logger.h" +#include "saitypes.h" +#include "unordered_map" +#include "dbconnector.h" +#include "set" +#include "orch.h" +#include "crmorch.h" +#include "nexthopgroupkey.h" +#include "bulker.h" + +using namespace std; + +extern sai_object_id_t gSwitchId; + +extern CrmOrch *gCrmOrch; + +extern sai_next_hop_group_api_t* sai_next_hop_group_api; +extern size_t gMaxBulkSize; + +/* + * Base class for next hop groups, containing the common interface that every + * next hop group should have based on what RouteOrch needs when working with + * next hop groups. + */ +class NhgBase +{ +public: + NhgBase() : m_id(SAI_NULL_OBJECT_ID) { SWSS_LOG_ENTER(); } + + NhgBase(NhgBase &&nhg) : m_id(nhg.m_id) + { SWSS_LOG_ENTER(); nhg.m_id = SAI_NULL_OBJECT_ID; } + + NhgBase& operator=(NhgBase &&nhg) + { SWSS_LOG_ENTER(); swap(m_id, nhg.m_id); return *this; } + + /* + * Prevent copying. + */ + NhgBase(const NhgBase&) = delete; + void operator=(const NhgBase&) = delete; + + virtual ~NhgBase(); + + /* + * Getters. + */ + inline sai_object_id_t getId() const { SWSS_LOG_ENTER(); return m_id; } + static inline unsigned getSyncedCount() + { SWSS_LOG_ENTER(); return m_syncedCount; } + + /* + * Check if the next hop group is synced or not. + */ + inline bool isSynced() const { return m_id != SAI_NULL_OBJECT_ID; } + + /* + * Check if the next hop group is temporary. + */ + virtual bool isTemp() const = 0; + + /* + * Get the NextHopGroupKey of this object. + */ + virtual NextHopGroupKey getNhgKey() const = 0; + + /* Increment the number of existing groups. */ + static inline void incSyncedCount() { SWSS_LOG_ENTER(); ++m_syncedCount; } + + /* Decrement the number of existing groups. */ + static void decSyncedCount(); + +protected: + /* + * The SAI ID of this object. + */ + sai_object_id_t m_id; + + /* + * Number of synced NHGs. Incremented when an object is synced and + * decremented when an object is removed. This will also account for the + * groups created by RouteOrch. + */ + static unsigned m_syncedCount; +}; + +/* + * NhgMember class representing the common templated base class between + * WeightedNhgMember and CbfNhgMember classes. + */ +template +class NhgMember +{ +public: + explicit NhgMember(const Key &key) : + m_key(key), m_id(SAI_NULL_OBJECT_ID) { SWSS_LOG_ENTER(); } + + NhgMember(NhgMember &&nhgm) : m_key(move(nhgm.m_key)), m_id(nhgm.m_id) + { SWSS_LOG_ENTER(); nhgm.m_id = SAI_NULL_OBJECT_ID; } + + NhgMember& operator=(NhgMember &&nhgm) + { + SWSS_LOG_ENTER(); + + swap(m_key, nhgm.m_key); + swap(m_id, nhgm.m_id); + + return *this; + } + + /* + * Prevent copying. + */ + NhgMember(const NhgMember&) = delete; + void operator=(const NhgMember&) = delete; + + virtual ~NhgMember() + { + SWSS_LOG_ENTER(); + + if (isSynced()) + { + SWSS_LOG_ERROR("Deleting next hop group member which is still synced"); + assert(false); + } + } + + /* + * Sync the NHG member, setting its SAI ID. + */ + virtual void sync(sai_object_id_t gm_id) + { + SWSS_LOG_ENTER(); + + /* + * The SAI ID should be updated from invalid to something valid. + */ + if ((m_id != SAI_NULL_OBJECT_ID) || (gm_id == SAI_NULL_OBJECT_ID)) + { + SWSS_LOG_ERROR("Setting invalid SAI ID %lu to next hop group " + "membeer %s, with current SAI ID %lu", + gm_id, to_string().c_str(), m_id); + throw logic_error("Invalid SAI ID assigned to next hop group member"); + } + + m_id = gm_id; + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + } + + /* + * Remove the group member, resetting its SAI ID. + */ + virtual void remove() + { + SWSS_LOG_ENTER(); + + /* + * If the membeer is not synced, exit. + */ + if (!isSynced()) + { + return; + } + + m_id = SAI_NULL_OBJECT_ID; + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + } + + /* + * Getters. + */ + inline Key getKey() const { return m_key; } + inline sai_object_id_t getId() const { return m_id; } + + /* + * Check whether the group is synced. + */ + inline bool isSynced() const { return m_id != SAI_NULL_OBJECT_ID; } + + /* + * Get a string form of the member. + */ + virtual string to_string() const = 0; + +protected: + /* + * The index / key of this NHG member. + */ + Key m_key; + + /* + * The SAI ID of this NHG member. + */ + sai_object_id_t m_id; +}; + +/* + * NhgCommon class representing the common templated base class between + * Nhg and CbfNhg classes. + */ +template +class NhgCommon : public NhgBase +{ +public: + /* + * Constructors. + */ + explicit NhgCommon(const Key &key) : m_key(key) { SWSS_LOG_ENTER(); } + + NhgCommon(NhgCommon &&nhg) : NhgBase(move(nhg)), + m_key(move(nhg.m_key)), + m_members(move(nhg.m_members)) + { SWSS_LOG_ENTER(); } + + NhgCommon& operator=(NhgCommon &&nhg) + { + SWSS_LOG_ENTER(); + + swap(m_key, nhg.m_key); + swap(m_members, nhg.m_members); + + NhgBase::operator=(move(nhg)); + + return *this; + } + + /* + * Check if the group contains the given member. + */ + inline bool hasMember(const MbrKey &key) const + { SWSS_LOG_ENTER(); return m_members.find(key) != m_members.end(); } + + /* + * Getters. + */ + inline Key getKey() const { SWSS_LOG_ENTER(); return m_key; } + inline size_t getSize() const + { SWSS_LOG_ENTER(); return m_members.size(); } + + /* + * Sync the group, generating a SAI ID. + */ + virtual bool sync() = 0; + + /* + * Remove the group, releasing the SAI ID. + */ + virtual bool remove() + { + SWSS_LOG_ENTER(); + + /* + * If the group is already removed, there is nothing to be done. + */ + if (!isSynced()) + { + return true; + } + + /* + * Remove the group members. + */ + set members; + + for (const auto &member : m_members) + { + members.insert(member.first); + } + + if (!removeMembers(members)) + { + SWSS_LOG_ERROR("Failed to remove next hop group %s members", + to_string().c_str()); + return false; + } + + /* + * Remove the NHG over SAI. + */ + auto status = sai_next_hop_group_api->remove_next_hop_group(m_id); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove next hop group %s, rv: %d", + to_string().c_str(), status); + return false; + } + + /* + * Decrease the number of programmed NHGs. + */ + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); + decSyncedCount(); + + /* + * Reset the group ID. + */ + m_id = SAI_NULL_OBJECT_ID; + + return true; + } + + /* + * Get a string representation of this next hop group. + */ + virtual string to_string() const = 0; + +protected: + /* + * The key indexing this object. + */ + Key m_key; + + /* + * The members of this group. + */ + map m_members; + + /* + * Sync the given members in the group. + */ + virtual bool syncMembers(const set &member_keys) = 0; + + /* + * Remove the given members from the group. + */ + virtual bool removeMembers(const set &member_keys) + { + SWSS_LOG_ENTER(); + + /* + * Remove all the given members from the group. + */ + ObjectBulker bulker(sai_next_hop_group_api, + gSwitchId, + gMaxBulkSize); + map statuses; + + for (const auto &key : member_keys) + { + const auto &nhgm = m_members.at(key); + + if (nhgm.isSynced()) + { + bulker.remove_entry(&statuses[key], nhgm.getId()); + } + } + + /* + * Flush the bulker to remove the members. + */ + bulker.flush(); + + /* + * Iterate over the returned statuses and check if the removal was + * successful. If it was, remove the member, otherwise log an error + * message. + */ + bool success = true; + + for (const auto &status : statuses) + { + auto &member = m_members.at(status.first); + + if (status.second == SAI_STATUS_SUCCESS) + { + member.remove(); + } + else + { + SWSS_LOG_ERROR("Failed to remove next hop group member %s, rv: %d", + member.to_string().c_str(), + status.second); + success = false; + } + } + + return success; + } + + /* + * Get the SAI attributes for creating a next hop group member over SAI. + */ + virtual vector createNhgmAttrs(const Mbr &member) + const = 0; +}; + +/* + * Structure describing a next hop group which NhgHandler owns. Beside having + * a next hop group, we also want to keep a ref count so we don't delete + * objects that are still referenced. + */ +template +struct NhgEntry +{ + /* The next hop group object in this entry. */ + NhgClass nhg; + + /* Number of external objects referencing this next hop group. */ + unsigned ref_count; + + NhgEntry() = default; + explicit NhgEntry(NhgClass&& _nhg, unsigned int _ref_count = 0) : + nhg(move(_nhg)), ref_count(_ref_count) { SWSS_LOG_ENTER(); } +}; + +/* + * Class providing the common functionality shared by all NhgHandler classes. + */ +template +class NhgHandlerCommon +{ +public: + /* + * Check if the given next hop group index exists. + */ + inline bool hasNhg(const string &index) const + { + SWSS_LOG_ENTER(); + return m_syncedNextHopGroups.find(index) != m_syncedNextHopGroups.end(); + } + + /* + * Get the next hop group with the given index. If the index does not + * exist in the map, a out_of_range eexception will be thrown. + */ + inline const NhgClass& getNhg(const string &index) const + { SWSS_LOG_ENTER(); return m_syncedNextHopGroups.at(index).nhg; } + + /* Increase the ref count for a NHG given by it's index. */ + void incNhgRefCount(const string& index) + { + SWSS_LOG_ENTER(); + + auto& nhg_entry = m_syncedNextHopGroups.at(index); + ++nhg_entry.ref_count; + } + + /* Dencrease the ref count for a NHG given by it's index. */ + void decNhgRefCount(const string& index) + { + SWSS_LOG_ENTER(); + + auto& nhg_entry = m_syncedNextHopGroups.at(index); + + /* Sanity check so we don't overflow. */ + if (nhg_entry.ref_count == 0) + { + SWSS_LOG_ERROR("Trying to decrement next hop group %s reference " + "count while none are left.", + nhg_entry.nhg.to_string().c_str()); + throw logic_error("Decreasing ref count which is already 0"); + } + + --nhg_entry.ref_count; + } + +protected: + /* + * Map of synced next hop groups. + */ + unordered_map> m_syncedNextHopGroups; +}; diff --git a/orchagent/nhgorch.cpp b/orchagent/nhghandler.cpp similarity index 71% rename from orchagent/nhgorch.cpp rename to orchagent/nhghandler.cpp index 4b2084e82a..510ff5a763 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhghandler.cpp @@ -1,15 +1,14 @@ +#include "nhghandler.h" #include "nhgorch.h" #include "neighorch.h" -#include "crmorch.h" #include "routeorch.h" +#include "crmorch.h" #include "bulker.h" #include "logger.h" #include "swssnet.h" extern sai_object_id_t gSwitchId; -extern PortsOrch *gPortsOrch; -extern CrmOrch *gCrmOrch; extern NeighOrch *gNeighOrch; extern RouteOrch *gRouteOrch; extern NhgOrch *gNhgOrch; @@ -18,15 +17,6 @@ extern size_t gMaxBulkSize; extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern sai_next_hop_api_t* sai_next_hop_api; -extern sai_switch_api_t* sai_switch_api; - -unsigned int NextHopGroup::m_count = 0; - -NhgOrch::NhgOrch(DBConnector *db, string tableName) : - Orch(db, tableName) -{ - SWSS_LOG_ENTER(); -} /* * Purpose: Perform the operations requested by APPL_DB users. @@ -37,15 +27,10 @@ NhgOrch::NhgOrch(DBConnector *db, string tableName) : * Params: IN consumer - The cosumer object. * Returns: Nothing. */ -void NhgOrch::doTask(Consumer& consumer) +void NhgHandler::doTask(Consumer& consumer) { SWSS_LOG_ENTER(); - if (!gPortsOrch->allPortsReady()) - { - return; - } - auto it = consumer.m_toSync.begin(); while (it != consumer.m_toSync.end()) @@ -56,7 +41,7 @@ void NhgOrch::doTask(Consumer& consumer) string op = kfvOp(t); bool success = false; - const auto& nhg_it = m_syncdNextHopGroups.find(index); + const auto& nhg_it = m_syncedNextHopGroups.find(index); if (op == SET_COMMAND) { @@ -102,7 +87,7 @@ void NhgOrch::doTask(Consumer& consumer) NextHopGroupKey nhg_key = NextHopGroupKey(nhg_str, weights); /* If the group does not exist, create one. */ - if (nhg_it == m_syncdNextHopGroups.end()) + if (nhg_it == m_syncedNextHopGroups.end()) { /* * If we've reached the NHG limit, we're going to create a temporary @@ -111,16 +96,16 @@ void NhgOrch::doTask(Consumer& consumer) * to be kept in the sync list so we keep trying to create the * actual group when there are enough resources. */ - if (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount()) + if (gRouteOrch->getNhgCount() + Nhg::getSyncedCount() >= gRouteOrch->getMaxNhgCount()) { SWSS_LOG_DEBUG("Next hop group count reached its limit."); try { - auto nhg = std::make_unique(createTempNhg(nhg_key)); - if (nhg->sync()) + auto nhg = createTempNhg(nhg_key); + if (nhg.sync()) { - m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); + m_syncedNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } else { @@ -138,19 +123,19 @@ void NhgOrch::doTask(Consumer& consumer) } else { - auto nhg = std::make_unique(nhg_key, false); - success = nhg->sync(); + auto nhg = Nhg(nhg_key, false); + success = nhg.sync(); if (success) { - m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); + m_syncedNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } } } /* If the group exists, update it. */ else { - const auto& nhg_ptr = nhg_it->second.nhg; + auto& nhg = nhg_it->second.nhg; /* * If the update would mandate promoting a temporary next hop @@ -158,8 +143,8 @@ void NhgOrch::doTask(Consumer& consumer) * resources yet, we have to skip it until we have enough * resources. */ - if (nhg_ptr->isTemp() && - (gRouteOrch->getNhgCount() + NextHopGroup::getCount() >= gRouteOrch->getMaxNhgCount())) + if (nhg.isTemp() && + (gRouteOrch->getNhgCount() + Nhg::getSyncedCount() >= gRouteOrch->getMaxNhgCount())) { /* * If the group was updated in such way that the previously @@ -168,12 +153,12 @@ void NhgOrch::doTask(Consumer& consumer) * the new key. Otherwise, this will be a no-op as we have * to wait for resources in order to promote the group. */ - if (!nhg_key.contains(nhg_ptr->getKey())) + if (!nhg_key.contains(nhg.getKey())) { try { /* Create the new temporary next hop group. */ - auto new_nhg = std::make_unique(createTempNhg(nhg_key)); + auto new_nhg = createTempNhg(nhg_key); /* * If we successfully sync the new group, update @@ -181,7 +166,7 @@ void NhgOrch::doTask(Consumer& consumer) * don't mess up the reference counter, as other * objects may already reference it. */ - if (new_nhg->sync()) + if (new_nhg.sync()) { nhg_it->second.nhg = std::move(new_nhg); } @@ -204,10 +189,10 @@ void NhgOrch::doTask(Consumer& consumer) * If the group is temporary but can now be promoted, create and sync a new group for * the desired next hops. */ - else if (nhg_ptr->isTemp()) + else if (nhg.isTemp()) { - auto nhg = std::make_unique(nhg_key, false); - success = nhg->sync(); + auto nhg = Nhg(nhg_key, false); + success = nhg.sync(); if (success) { @@ -221,14 +206,26 @@ void NhgOrch::doTask(Consumer& consumer) /* Common update, when all the requirements are met. */ else { - success = nhg_ptr->update(nhg_key); + success = nhg.update(nhg_key); } } } else if (op == DEL_COMMAND) { + /* + * If there is a pending SET after this DEL operation, skip the + * DEL operation to perform the update instead. Otherwise, in the + * scenario where the DEL operation may be blocked by the ref + * counter, we'd end up deleting the object after the SET operation + * is performed, which would not reflect the desired state of the + * object. + */ + if (consumer.m_toSync.count(it->first) > 1) + { + success = true; + } /* If the group does not exist, do nothing. */ - if (nhg_it == m_syncdNextHopGroups.end()) + else if (nhg_it == m_syncedNextHopGroups.end()) { SWSS_LOG_INFO("Unable to find group with key %s to remove", index.c_str()); /* Mark the operation as successful to consume it. */ @@ -242,19 +239,19 @@ void NhgOrch::doTask(Consumer& consumer) /* Else, if the group is no more referenced, remove it. */ else { - const auto& nhg = nhg_it->second.nhg; + auto& nhg = nhg_it->second.nhg; - success = nhg->remove(); + success = nhg.remove(); if (success) { - m_syncdNextHopGroups.erase(nhg_it); + m_syncedNextHopGroups.erase(nhg_it); } } } else { - SWSS_LOG_ERROR("Unknown operation type %s\n", op.c_str()); + SWSS_LOG_WARN("Unknown operation type %s", op.c_str()); /* Mark the operation as successful to consume it. */ success = true; } @@ -280,7 +277,7 @@ void NhgOrch::doTask(Consumer& consumer) * containing groups; * false, otherwise. */ -bool NhgOrch::validateNextHop(const NextHopKey& nh_key) +bool NhgHandler::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); @@ -288,17 +285,17 @@ bool NhgOrch::validateNextHop(const NextHopKey& nh_key) * Iterate through all groups and validate the next hop in those who * contain it. */ - for (auto& it : m_syncdNextHopGroups) + for (auto& it : m_syncedNextHopGroups) { auto& nhg = it.second.nhg; - if (nhg->hasNextHop(nh_key)) + if (nhg.hasMember(nh_key)) { /* * If sync fails, exit right away, as we expect it to be due to a * raeson for which any other future validations will fail too. */ - if (!nhg->validateNextHop(nh_key)) + if (!nhg.validateNextHop(nh_key)) { SWSS_LOG_ERROR("Failed to validate next hop %s in group %s", nh_key.to_string().c_str(), @@ -320,7 +317,7 @@ bool NhgOrch::validateNextHop(const NextHopKey& nh_key) * containing groups; * false, otherwise. */ -bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) +bool NhgHandler::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); @@ -328,14 +325,14 @@ bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) * Iterate through all groups and invalidate the next hop from those who * contain it. */ - for (auto& it : m_syncdNextHopGroups) + for (auto& it : m_syncedNextHopGroups) { auto& nhg = it.second.nhg; - if (nhg->hasNextHop(nh_key)) + if (nhg.hasMember(nh_key)) { /* If the remove fails, exit right away. */ - if (!nhg->invalidateNextHop(nh_key)) + if (!nhg.invalidateNextHop(nh_key)) { SWSS_LOG_WARN("Failed to invalidate next hop %s from group %s", nh_key.to_string().c_str(), @@ -348,37 +345,6 @@ bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) return true; } -/* - * Purpose: Increase the ref count for a next hop group. - * Description: Increment the ref count for a next hop group by 1. - * Params: IN index - The index of the next hop group. - * Returns: Nothing. - */ -void NhgOrch::incNhgRefCount(const std::string& index) -{ - SWSS_LOG_ENTER(); - - NhgEntry& nhg_entry = m_syncdNextHopGroups.at(index); - ++nhg_entry.ref_count; -} - -/* - * Purpose: Decrease the ref count for a next hop group. - * Description: Decrement the ref count for a next hop group by 1. - * Params: IN index - The index of the next hop group. - * Returns: Nothing. - */ -void NhgOrch::decNhgRefCount(const std::string& index) -{ - SWSS_LOG_ENTER(); - - NhgEntry& nhg_entry = m_syncdNextHopGroups.at(index); - - /* Sanity check so we don't overflow. */ - assert(nhg_entry.ref_count > 0); - --nhg_entry.ref_count; -} - /* * Purpose: Get the next hop ID of the member. * Description: Get the SAI ID of the next hop from NeighOrch. @@ -386,15 +352,15 @@ void NhgOrch::decNhgRefCount(const std::string& index) * Returns: The SAI ID of the next hop, or SAI_NULL_OBJECT_ID if the next * hop is not valid. */ -sai_object_id_t NextHopGroupMember::getNhId() const +sai_object_id_t WeightedNhgMember::getNhId() const { SWSS_LOG_ENTER(); sai_object_id_t nh_id = SAI_NULL_OBJECT_ID; - if (gNeighOrch->hasNextHop(m_nh_key)) + if (gNeighOrch->hasNextHop(m_key)) { - nh_id = gNeighOrch->getNextHopId(m_nh_key); + nh_id = gNeighOrch->getNextHopId(m_key); } /* * If the next hop is labeled and the IP next hop exists, create the @@ -403,35 +369,19 @@ sai_object_id_t NextHopGroupMember::getNhId() const * after the object is created and would never create the labeled next hop * afterwards. */ - else if (isLabeled() && gNeighOrch->isNeighborResolved(m_nh_key)) + else if (isLabeled() && gNeighOrch->isNeighborResolved(m_key)) { - gNeighOrch->addNextHop(m_nh_key); - nh_id = gNeighOrch->getNextHopId(m_nh_key); + gNeighOrch->addNextHop(m_key); + nh_id = gNeighOrch->getNextHopId(m_key); } else { - gNeighOrch->resolveNeighbor(m_nh_key); + gNeighOrch->resolveNeighbor(m_key); } return nh_id; } -/* - * Purpose: Move assignment operator. - * Description: Perform member-wise swap. - * Params: IN nhgm - The next hop group member to swap. - * Returns: Reference to this object. - */ -NextHopGroupMember& NextHopGroupMember::operator=(NextHopGroupMember&& nhgm) -{ - SWSS_LOG_ENTER(); - - std::swap(m_nh_key, nhgm.m_nh_key); - std::swap(m_gm_id, nhgm.m_gm_id); - - return *this; -} - /* * Purpose: Update the weight of a member. * Description: Set the new member's weight and if the member is synced, update @@ -440,21 +390,21 @@ NextHopGroupMember& NextHopGroupMember::operator=(NextHopGroupMember&& nhgm) * Returns: true, if the operation was successful; * false, otherwise. */ -bool NextHopGroupMember::updateWeight(uint32_t weight) +bool WeightedNhgMember::updateWeight(uint32_t weight) { SWSS_LOG_ENTER(); bool success = true; - m_nh_key.weight = weight; + m_key.weight = weight; if (isSynced()) { sai_attribute_t nhgm_attr; nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; - nhgm_attr.value.s32 = m_nh_key.weight; + nhgm_attr.value.s32 = m_key.weight; - sai_status_t status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_gm_id, &nhgm_attr); + sai_status_t status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_id, &nhgm_attr); success = status == SAI_STATUS_SUCCESS; } @@ -468,16 +418,12 @@ bool NextHopGroupMember::updateWeight(uint32_t weight) * Params: IN gm_id - The group member SAI ID to set. * Returns: Nothing. */ -void NextHopGroupMember::sync(sai_object_id_t gm_id) +void WeightedNhgMember::sync(sai_object_id_t gm_id) { SWSS_LOG_ENTER(); - /* The SAI ID should be updated from invalid to something valid. */ - assert((m_gm_id == SAI_NULL_OBJECT_ID) && (gm_id != SAI_NULL_OBJECT_ID)); - - m_gm_id = gm_id; - gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); - gNeighOrch->increaseNextHopRefCount(m_nh_key); + NhgMember::sync(gm_id); + gNeighOrch->increaseNextHopRefCount(m_key); } /* @@ -487,22 +433,12 @@ void NextHopGroupMember::sync(sai_object_id_t gm_id) * Params: None. * Returns: Nothing. */ -void NextHopGroupMember::remove() +void WeightedNhgMember::remove() { SWSS_LOG_ENTER(); - /* - * If the member is already removed, exit so we don't decrement the ref - * counters more than once. - */ - if (!isSynced()) - { - return; - } - - m_gm_id = SAI_NULL_OBJECT_ID; - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); - gNeighOrch->decreaseNextHopRefCount(m_nh_key); + NhgMember::remove(); + gNeighOrch->decreaseNextHopRefCount(m_key); } /* @@ -512,15 +448,10 @@ void NextHopGroupMember::remove() * Params: None. * Returns: Nothing. */ -NextHopGroupMember::~NextHopGroupMember() +WeightedNhgMember::~WeightedNhgMember() { SWSS_LOG_ENTER(); - /* - * The group member should be removed from its group before destroying it. - */ - assert(!isSynced()); - /* * If the labeled next hop is unreferenced, remove it from NeighOrch as * NhgOrch and RouteOrch are the ones controlling it's lifetime. They both @@ -529,10 +460,10 @@ NextHopGroupMember::~NextHopGroupMember() * next hop. */ if (isLabeled() && - gNeighOrch->hasNextHop(m_nh_key) && - (gNeighOrch->getNextHopRefCount(m_nh_key) == 0)) + gNeighOrch->hasNextHop(m_key) && + (gNeighOrch->getNextHopRefCount(m_key) == 0)) { - gNeighOrch->removeMplsNextHop(m_nh_key); + gNeighOrch->removeMplsNextHop(m_key); } } @@ -542,53 +473,31 @@ NextHopGroupMember::~NextHopGroupMember() * Params: IN key - The next hop group's key. * Returns: Nothing. */ -NextHopGroup::NextHopGroup(const NextHopGroupKey& key, bool is_temp) : - m_key(key), - m_id(SAI_NULL_OBJECT_ID), - m_is_temp(is_temp) +Nhg::Nhg(const NextHopGroupKey& key, bool is_temp) : NhgCommon(key), m_is_temp(is_temp) { SWSS_LOG_ENTER(); /* Parse the key and create the members. */ - for (const auto& it : m_key.getNextHops()) + for (const auto &it : m_key.getNextHops()) { - m_members.emplace(it, NextHopGroupMember(it)); + m_members.emplace(it, WeightedNhgMember(it)); } } -/* - * Purpose: Move constructor. - * Description: Initialize the members by doing member-wise move construct. - * Params: IN nhg - The rvalue object to initialize from. - * Returns: Nothing. - */ -NextHopGroup::NextHopGroup(NextHopGroup&& nhg) : - m_key(std::move(nhg.m_key)), - m_id(std::move(nhg.m_id)), - m_members(std::move(nhg.m_members)), - m_is_temp(nhg.m_is_temp) -{ - SWSS_LOG_ENTER(); - - /* Invalidate the rvalue SAI ID. */ - nhg.m_id = SAI_NULL_OBJECT_ID; -} - /* * Purpose: Move assignment operator. * Description: Perform member-wise swap. * Params: IN nhg - The rvalue object to swap with. * Returns: Referene to this object. */ -NextHopGroup& NextHopGroup::operator=(NextHopGroup&& nhg) +Nhg& Nhg::operator=(Nhg&& nhg) { SWSS_LOG_ENTER(); - std::swap(m_key, nhg.m_key); - std::swap(m_id, nhg.m_id); - std::swap(m_members, nhg.m_members); m_is_temp = nhg.m_is_temp; + NhgCommon::operator=(std::move(nhg)); + return *this; } @@ -602,7 +511,7 @@ NextHopGroup& NextHopGroup::operator=(NextHopGroup&& nhg) * Returns: true, if the operation was successful; * false, otherwise. */ -bool NextHopGroup::sync() +bool Nhg::sync() { SWSS_LOG_ENTER(); @@ -618,12 +527,12 @@ bool NextHopGroup::sync() */ if (m_is_temp) { - const NextHopGroupMember& nhgm = m_members.begin()->second; + const WeightedNhgMember& nhgm = m_members.begin()->second; sai_object_id_t nhid = nhgm.getNhId(); if (nhid == SAI_NULL_OBJECT_ID) { - SWSS_LOG_WARN("Next hop %s is not synced", nhgm.getNhKey().to_string().c_str()); + SWSS_LOG_WARN("Next hop %s is not synced", nhgm.getKey().to_string().c_str()); return false; } else @@ -667,7 +576,7 @@ bool NextHopGroup::sync() gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); /* Increment the number of synced NHGs. */ - ++m_count; + ++m_syncedCount; /* * Try creating the next hop group's members over SAI. @@ -693,7 +602,7 @@ bool NextHopGroup::sync() * Params: IN index - The CP index of the next hop group. * Returns: The created temporary next hop group. */ -NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) +Nhg NhgHandler::createTempNhg(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); @@ -726,7 +635,7 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) advance(it, rand() % valid_nhs.size()); /* Create the temporary group. */ - NextHopGroup nhg(NextHopGroupKey(it->to_string()), true); + Nhg nhg(NextHopGroupKey(it->to_string()), true); return nhg; } @@ -739,51 +648,18 @@ NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) * Returns: true, if the operation was successful; * false, otherwise */ -bool NextHopGroup::remove() +bool Nhg::remove() { SWSS_LOG_ENTER(); - /* If the group is already removed, or is temporary, there is nothing to be done - - * just reset the ID. - */ - if (!isSynced() || m_is_temp) + // If the group is temporary, there is nothing to be done - just reset the ID. + if (m_is_temp) { m_id = SAI_NULL_OBJECT_ID; return true; } - /* Remove group's members. If we failed to remove the members, exit. */ - if (!removeMembers(m_key.getNextHops())) - { - SWSS_LOG_ERROR("Failed to remove group %s members", to_string().c_str()); - return false; - } - - /* Remove the group. */ - sai_status_t status = sai_next_hop_group_api-> - remove_next_hop_group(m_id); - - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to remove next hop group %s, rv: %d", - m_key.to_string().c_str(), status); - - task_process_status handle_status = gNhgOrch->handleSaiRemoveStatus(SAI_API_NEXT_HOP_GROUP, status); - if (handle_status != task_success) - { - return gNhgOrch->parseHandleSaiStatusFailure(handle_status); - } - } - - /* If the operation is successful, release the resources. */ - gCrmOrch->decCrmResUsedCounter( - CrmResourceType::CRM_NEXTHOP_GROUP); - --m_count; - - /* Reset the group ID. */ - m_id = SAI_NULL_OBJECT_ID; - - return true; + return NhgCommon::remove(); } /* @@ -796,7 +672,7 @@ bool NextHopGroup::remove() * Returns: true, if the members were added succesfully; * false, otherwise. */ -bool NextHopGroup::syncMembers(const std::set& nh_keys) +bool Nhg::syncMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); @@ -813,10 +689,10 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) for (const auto& nh_key : nh_keys) { - NextHopGroupMember& nhgm = m_members.at(nh_key); + WeightedNhgMember& nhgm = m_members.at(nh_key); /* If the member is already synced, continue. */ - if (nhgm.getGmId() != SAI_NULL_OBJECT_ID) + if (nhgm.isSynced()) { continue; } @@ -873,66 +749,6 @@ bool NextHopGroup::syncMembers(const std::set& nh_keys) return success; } -/* - * Purpose: Remove the given group's members over the SAI API. - * Description: Go through the given members and remove them. - * Params: IN nh_keys - The next hop keys of the members to remove. - * Returns: true, if the operation was successful; - * false, otherwise - */ -bool NextHopGroup::removeMembers(const std::set& nh_keys) -{ - SWSS_LOG_ENTER(); - - ObjectBulker nextHopGroupMemberBulker( - sai_next_hop_group_api, gSwitchId, gMaxBulkSize); - - /* - * Iterate through the given group members add them to be removed. - * - * Keep track of the SAI remove statuses in case one of them returns an - * error. We assume that removal should always succeed. If for some - * reason it doesn't, there's nothing we can do, but we'll log an error - * later. - */ - std::map statuses; - - for (const auto& nh_key : nh_keys) - { - const NextHopGroupMember& nhgm = m_members.at(nh_key); - - if (nhgm.isSynced()) - { - nextHopGroupMemberBulker.remove_entry(&statuses[nh_key], nhgm.getGmId()); - } - } - - /* Flush the bulker to apply the changes. */ - nextHopGroupMemberBulker.flush(); - - /* - * Iterate over the statuses map and check if the removal was successful. - * If it was, decrement the Crm counter and reset the member's ID. If it - * wasn't, log an error message. - */ - bool success = true; - - for (const auto& status : statuses) - { - if (status.second == SAI_STATUS_SUCCESS) - { - m_members.at(status.first).remove(); - } - else - { - SWSS_LOG_ERROR("Could not remove next hop group member %s, rv: %d", - status.first.to_string().c_str(), status.second); - success = false; - } - } - - return success; -} /* * Purpose: Update the next hop group based on a new next hop group key. @@ -945,7 +761,7 @@ bool NextHopGroup::removeMembers(const std::set& nh_keys) * Returns: true, if the operation was successful; * false, otherwise. */ -bool NextHopGroup::update(const NextHopGroupKey& nhg_key) +bool Nhg::update(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); @@ -1001,7 +817,7 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) /* Add any new members to the group. */ for (const auto& it : new_nh_keys) { - m_members.emplace(it, NextHopGroupMember(it)); + m_members.emplace(it, WeightedNhgMember(it)); } /* @@ -1024,7 +840,7 @@ bool NextHopGroup::update(const NextHopGroupKey& nhg_key) * Params: IN nhgm - The next hop group member. * Returns: The attributes vector for the given next hop. */ -vector NextHopGroup::createNhgmAttrs(const NextHopGroupMember& nhgm) const +vector Nhg::createNhgmAttrs(const WeightedNhgMember& nhgm) const { SWSS_LOG_ENTER(); @@ -1056,7 +872,7 @@ vector NextHopGroup::createNhgmAttrs(const NextHopGroupMember& * Returns: true, if the operation was successful; * false, otherwise. */ -bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) +bool Nhg::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); @@ -1070,7 +886,7 @@ bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) * Returns: true, if the operation was successful; * false, otherwise. */ -bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) +bool Nhg::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); diff --git a/orchagent/nhghandler.h b/orchagent/nhghandler.h new file mode 100644 index 0000000000..301ef398a4 --- /dev/null +++ b/orchagent/nhghandler.h @@ -0,0 +1,116 @@ +#pragma once + +#include "nhgbase.h" + +using namespace std; + +class WeightedNhgMember : public NhgMember +{ +public: + /* Constructors / Assignment operators. */ + WeightedNhgMember(const NextHopKey& nh_key) : + NhgMember(nh_key) {} + + WeightedNhgMember(WeightedNhgMember&& nhgm) : + NhgMember(move(nhgm)) {} + + /* Destructor. */ + ~WeightedNhgMember(); + + /* Update member's weight and update the SAI attribute as well. */ + bool updateWeight(uint32_t weight); + + /* Sync / Remove. */ + void sync(sai_object_id_t gm_id) override; + void remove() override; + + /* Getters / Setters. */ + inline uint32_t getWeight() const { return m_key.weight; } + sai_object_id_t getNhId() const; + + /* Check if the next hop is labeled. */ + inline bool isLabeled() const { return !m_key.label_stack.empty(); } + + /* Convert member's details to string. */ + string to_string() const override + { + return m_key.to_string() + + ", SAI ID: " + std::to_string(m_id); + } +}; + +/* + * Nhg class representing a next hop group object. + */ +class Nhg : public NhgCommon +{ +public: + /* Constructors. */ + explicit Nhg(const NextHopGroupKey& key, bool is_temp); + + Nhg(Nhg&& nhg) : + NhgCommon(move(nhg)), m_is_temp(nhg.m_is_temp) + { SWSS_LOG_ENTER(); } + + Nhg& operator=(Nhg&& nhg); + + ~Nhg() { SWSS_LOG_ENTER(); remove(); } + + /* Sync the group, creating the group's and members SAI IDs. */ + bool sync() override; + + /* Remove the group, reseting the group's and members SAI IDs. */ + bool remove() override; + + /* + * Update the group based on a new next hop group key. This will also + * perform any sync / remove necessary. + */ + bool update(const NextHopGroupKey& nhg_key); + + /* Validate a next hop in the group, syncing it. */ + bool validateNextHop(const NextHopKey& nh_key); + + /* Invalidate a next hop in the group, removing it. */ + bool invalidateNextHop(const NextHopKey& nh_key); + + /* Getters / Setters. */ + inline bool isTemp() const override { return m_is_temp; } + inline void setTemp(bool is_temp) { m_is_temp = is_temp; } + + NextHopGroupKey getNhgKey() const override { return m_key; } + + /* Convert NHG's details to a string. */ + string to_string() const override + { + return m_key.to_string() + ", SAI ID: " + std::to_string(m_id); + } + +private: + /* Whether the group is temporary or not. */ + bool m_is_temp; + + /* Add group's members over the SAI API for the given keys. */ + bool syncMembers(const set& nh_keys) override; + + /* Create the attributes vector for a next hop group member. */ + vector createNhgmAttrs( + const WeightedNhgMember& nhgm) const override; +}; + +/* + * Next Hop Group Orchestrator class that handles NEXT_HOP_GROUP_TABLE + * updates. + */ +class NhgHandler : public NhgHandlerCommon +{ +public: + /* Add a temporary next hop group when resources are exhausted. */ + Nhg createTempNhg(const NextHopGroupKey& nhg_key); + + /* Validate / Invalidate a next hop. */ + bool validateNextHop(const NextHopKey& nh_key); + bool invalidateNextHop(const NextHopKey& nh_key); + + void doTask(Consumer &consumer); +}; diff --git a/orchagent/nhgmaporch.cpp b/orchagent/nhgmaporch.cpp new file mode 100644 index 0000000000..45604dd8a7 --- /dev/null +++ b/orchagent/nhgmaporch.cpp @@ -0,0 +1,366 @@ +#include "nhgmaporch.h" +#include "climits" + +extern sai_object_id_t gSwitchId; +extern sai_next_hop_group_api_t* sai_next_hop_group_api; +extern sai_switch_api_t *sai_switch_api; + +uint64_t NhgMapOrch::m_max_nhg_map_count = 0; + +NhgMapEntry::NhgMapEntry(sai_object_id_t _id, uint32_t _ref_count) : id(_id), ref_count(_ref_count) +{ + SWSS_LOG_ENTER(); +} + +NhgMapOrch::NhgMapOrch(swss::DBConnector *db, const string &table_name) : Orch(db, table_name) +{ + SWSS_LOG_ENTER(); + + /* + * Get the maximum number of NHG maps. + */ + if (sai_object_type_get_availability(gSwitchId, + SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MAP, + 0, + nullptr, + &m_max_nhg_map_count) != SAI_STATUS_SUCCESS) + { + SWSS_LOG_WARN("Switch does not support NHG maps"); + m_max_nhg_map_count = 0; + } + + SWSS_LOG_INFO("Maximum number of NHG maps: %lu", m_max_nhg_map_count); +} + +void NhgMapOrch::doTask(Consumer &consumer) +{ + SWSS_LOG_ENTER(); + + auto it = consumer.m_toSync.begin(); + while (it != consumer.m_toSync.end()) + { + swss::KeyOpFieldsValuesTuple t = it->second; + string index = kfvKey(t); + string op = kfvOp(t); + bool success = true; + + auto fc_map_it = m_syncdMaps.find(index); + + /* + * Set operation. + */ + if (op == SET_COMMAND) + { + /* + * Extract the map from the values. + */ + auto p = getMap(kfvFieldsValues(t)); + success = p.first; + + /* + * If the map can't be extracted, erase the work item as it's useless to retry unless the user updates the + * wrong values. We achieve the erase by setting the success value to "true". + * + */ + if (!success) + { + SWSS_LOG_ERROR("Failed to extract NHG map %s", index.c_str()); + success = true; + } + else + { + /* + * Create the SAI map. + */ + auto *fc_map = new sai_map_t[p.second.size()]; + uint32_t ii = 0; + + for (const auto &fc_nh_idx : p.second) + { + fc_map[ii].key = fc_nh_idx.first; + fc_map[ii++].value = fc_nh_idx.second; + } + + sai_map_list_t fc_map_list; + assert(p.second.size() <= UINT32_MAX); + fc_map_list.count = static_cast(p.second.size()); + fc_map_list.list = fc_map; + + /* + * Create the map. + */ + if (fc_map_it == m_syncdMaps.end()) + { + /* + * Check if we have enough resources for a new map. + */ + if (m_syncdMaps.size() >= m_max_nhg_map_count) + { + SWSS_LOG_WARN("No more resources left for new next hop group map %s", index.c_str()); + success = false; + } + else + { + /* + * Set the map attributes. + */ + sai_attribute_t attr; + vector attrs; + + attr.id = SAI_NEXT_HOP_GROUP_MAP_ATTR_TYPE; + attr.value.u32 = SAI_NEXT_HOP_GROUP_MAP_TYPE_FORWARDING_CLASS_TO_INDEX; + attrs.push_back(move(attr)); + + attr.id = SAI_NEXT_HOP_GROUP_MAP_ATTR_MAP_TO_VALUE_LIST; + attr.value.maplist = fc_map_list; + attrs.push_back(move(attr)); + + /* + * Create the map over SAI. + */ + sai_object_id_t nhg_map_id; + sai_status_t status = sai_next_hop_group_api->create_next_hop_group_map(&nhg_map_id, + gSwitchId, + static_cast(attrs.size()), + attrs.data()); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create NHG map %s, rv %d", index.c_str(), status); + success = false; + } + else + { + assert(nhg_map_id != SAI_NULL_OBJECT_ID); + m_syncdMaps.emplace(move(index), nhg_map_id); + } + } + } + /* + * Update the map. + */ + else + { + /* + * Update the map attribute. + */ + sai_attribute_t attr; + attr.id = SAI_NEXT_HOP_GROUP_MAP_ATTR_MAP_TO_VALUE_LIST; + attr.value.maplist = fc_map_list; + + sai_status_t status = sai_next_hop_group_api->set_next_hop_group_map_attribute(fc_map_it->second.id, + &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to update NHG map %s, rv %d", index.c_str(), status); + success = false; + } + } + + /* + * Free the allocated memory. + */ + delete[] fc_map; + } + } + /* + * Del operation. + */ + else if (op == DEL_COMMAND) + { + /* + * If there is a pending SET after this DEL operation, skip the + * DEL operation to perform the update instead. Otherwise, in the + * scenario where the DEL operation may be blocked by the ref + * counter, we'd end up deleting the object after the SET operation + * is performed, which would not reflect the desired state of the + * object. + */ + if (consumer.m_toSync.count(it->first) > 1) + { + success = true; + } + else if (fc_map_it == m_syncdMaps.end()) + { + SWSS_LOG_WARN("NHG map %s does not exist to deleted", index.c_str()); + } + else if (fc_map_it->second.ref_count > 0) + { + SWSS_LOG_WARN("Can not delete referenced NHG map %s", index.c_str()); + success = false; + } + else + { + sai_status_t status = sai_next_hop_group_api->remove_next_hop_group_map(fc_map_it->second.id); + + if (status == SAI_STATUS_SUCCESS) + { + m_syncdMaps.erase(fc_map_it); + } + else + { + SWSS_LOG_ERROR("Failed to remove NHG map %s, rv %d", index.c_str(), status); + success = false; + } + } + } + else + { + SWSS_LOG_WARN("Unknown operation type %s", op.c_str()); + + /* + * Mark the operation as a success to remove the task. + */ + success = true; + } + + /* + * Depending on the operation success, remove the task or skip it. + */ + if (success) + { + it = consumer.m_toSync.erase(it); + } + else + { + ++it; + } + } +} + +/* + * Return the SAI ID for the map indexed by "index". If it does not exist, return SAI_NULL_OBJECT_ID. + */ +sai_object_id_t NhgMapOrch::getMapId(const string &index) const +{ + SWSS_LOG_ENTER(); + + auto it = m_syncdMaps.find(index); + + return it == m_syncdMaps.end() ? SAI_NULL_OBJECT_ID : it->second.id; +} + +/* + * Increase reference counter for a map. + */ +void NhgMapOrch::incRefCount(const string &index) +{ + SWSS_LOG_ENTER(); + + ++m_syncdMaps.at(index).ref_count; +} + +/* + * Decrease reference counter for a map. + */ +void NhgMapOrch::decRefCount(const string &index) +{ + SWSS_LOG_ENTER(); + + auto &nhg_map = m_syncdMaps.at(index); + + if (nhg_map.ref_count == 0) + { + SWSS_LOG_ERROR("Decreasing reference counter beyond 0 for NHG map %s", index.c_str()); + throw std::runtime_error("Decreasing reference counter beyond 0"); + } + + --nhg_map.ref_count; +} + +/* + * Get the max FC value supported by the switch. + */ +sai_uint8_t NhgMapOrch::getMaxFcVal() +{ + SWSS_LOG_ENTER(); + + static int max_fc_val = -1; + + /* + * Get the maximum value allowed for FC if it wasn't already initialized. + */ + if (max_fc_val == -1) + { + sai_attribute_t attr; + attr.id = SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES; + + if (sai_switch_api->get_switch_attribute(gSwitchId, 1, &attr) == SAI_STATUS_SUCCESS) + { + max_fc_val = attr.value.u8; + } + else + { + SWSS_LOG_WARN("Switch does not support FCs"); + max_fc_val = 0; + } + } + + return static_cast(max_fc_val); +} + +/* + * Extract the NHG map from the FV tuples + */ +pair> NhgMapOrch::getMap(const vector &values) +{ + SWSS_LOG_ENTER(); + + bool success = true; + + /* + * If the map is empty, return error + */ + if (values.empty()) + { + SWSS_LOG_ERROR("NHG map is empty"); + success = false; + } + + unordered_map fc_map; + sai_uint8_t max_fc_val = getMaxFcVal(); + + /* + * Create the map while validating that the values are positive + */ + for (auto it = values.begin(); it != values.end(); ++it) + { + try + { + /* + * Check the FC value is valid. + */ + auto fc = stoi(fvField(*it)); + + if ((fc < 0) || (fc > max_fc_val)) + { + SWSS_LOG_ERROR("FC value %d is either negative or greater than max value %d", fc, max_fc_val); + success = false; + break; + } + + /* + * Check the NH index value is valid. + */ + auto nh_idx = stoi(fvValue(*it)); + + if (nh_idx < 0) + { + SWSS_LOG_ERROR("NH index %d is negative", nh_idx); + success = false; + break; + } + + fc_map.emplace(fc, nh_idx).second; + } + catch(const invalid_argument& e) + { + SWSS_LOG_ERROR("Got exception during conversion: %s", e.what()); + success = false; + break; + } + } + + return {success, fc_map}; +} diff --git a/orchagent/nhgmaporch.h b/orchagent/nhgmaporch.h new file mode 100644 index 0000000000..8e5d2b976a --- /dev/null +++ b/orchagent/nhgmaporch.h @@ -0,0 +1,61 @@ +#pragma once + +#include "orch.h" +#include "dbconnector.h" +#include + +using namespace std; + +/* + * Structure describing a NHG map entry in the NHG map map. + */ +struct NhgMapEntry +{ + /* The next hop group map SAI ID. */ + sai_object_id_t id; + + /* Number of external objects referencing this next hop group map. */ + uint32_t ref_count; + + explicit NhgMapEntry(sai_object_id_t id, uint32_t _ref_count = 0); +}; + +class NhgMapOrch : public Orch +{ +public: + NhgMapOrch(swss::DBConnector *db, const string &table_name); + + void doTask(Consumer &consumer) override; + + /* + * Return the SAI ID for the map indexed by "index". If it does not exist, return SAI_NULL_OBJECT_ID. + */ + sai_object_id_t getMapId(const string &key) const; + + /* + * Increase / Decrease reference counter for a map. + */ + void incRefCount(const string &key); + void decRefCount(const string &key); + + /* + * Get the max FC value supported by the switch. + */ + static sai_uint8_t getMaxFcVal(); + +private: + /* + * Map of synced NHG maps over SAI. + */ + unordered_map m_syncdMaps; + + /* + * Maximum number of NHG maps supported by the switch. + */ + static uint64_t m_max_nhg_map_count; + + /* + * Extract the NHG map from the FV tuples + */ + static pair> getMap(const vector &values); +}; diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index ce99ef85e3..20369c9a0e 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -1,229 +1,138 @@ #pragma once -#include "orch.h" -#include "nexthopgroupkey.h" +#include "cbfnhghandler.h" +#include "nhghandler.h" +#include "switchorch.h" +#include "vector" +#include "portsorch.h" -class NextHopGroupMember -{ -public: - /* Constructors / Assignment operators. */ - NextHopGroupMember(const NextHopKey& nh_key) : - m_nh_key(nh_key), - m_gm_id(SAI_NULL_OBJECT_ID) {} - - NextHopGroupMember(NextHopGroupMember&& nhgm) : - m_nh_key(std::move(nhgm.m_nh_key)), - m_gm_id(nhgm.m_gm_id) - { nhgm.m_gm_id = SAI_NULL_OBJECT_ID; } - - NextHopGroupMember& operator=(NextHopGroupMember&& nhgm); - - /* - * Prevent object copying so we don't end up having multiple objects - * referencing the same SAI objects. - */ - NextHopGroupMember(const NextHopGroupMember&) = delete; - void operator=(const NextHopGroupMember&) = delete; - - /* Destructor. */ - virtual ~NextHopGroupMember(); - - /* Update member's weight and update the SAI attribute as well. */ - bool updateWeight(uint32_t weight); +using namespace std; - /* Sync / Remove. */ - void sync(sai_object_id_t gm_id); - void remove(); +extern PortsOrch *gPortsOrch; - /* Getters / Setters. */ - inline const NextHopKey& getNhKey() const { return m_nh_key; } - inline uint32_t getWeight() const { return m_nh_key.weight; } - sai_object_id_t getNhId() const; - inline sai_object_id_t getGmId() const { return m_gm_id; } - inline bool isSynced() const { return m_gm_id != SAI_NULL_OBJECT_ID; } - - /* Check if the next hop is labeled. */ - inline bool isLabeled() const { return !m_nh_key.label_stack.empty(); } - - /* Convert member's details to string. */ - std::string to_string() const - { - return m_nh_key.to_string() + ", SAI ID: " + std::to_string(m_gm_id); - } - -private: - /* The key of the next hop of this member. */ - NextHopKey m_nh_key; - - /* The group member SAI ID for this member. */ - sai_object_id_t m_gm_id; -}; - -/* Map indexed by NextHopKey, containing the SAI ID of the group member. */ -typedef std::map NhgMembers; - -/* - * NextHopGroup class representing a next hop group object. - */ -class NextHopGroup +class NhgOrch : public Orch { public: - /* Constructors. */ - explicit NextHopGroup(const NextHopGroupKey& key, bool is_temp); - NextHopGroup(NextHopGroup&& nhg); - NextHopGroup& operator=(NextHopGroup&& nhg); - - /* Destructor. */ - virtual ~NextHopGroup() { remove(); } - - /* Sync the group, creating the group's and members SAI IDs. */ - bool sync(); + NhgOrch(DBConnector *db, const vector &table_names) : + Orch(db, table_names) + { + SWSS_LOG_ENTER(); + } - /* Remove the group, reseting the group's and members SAI IDs. */ - bool remove(); + /* + * Get the maximum number of ECMP groups allowed by the switch. + */ + static inline unsigned getMaxNhgCount() + { SWSS_LOG_ENTER(); return m_maxNhgCount; } /* - * Update the group based on a new next hop group key. This will also - * perform any sync / remove necessary. + * Get the number of next hop groups that are synced. */ - bool update(const NextHopGroupKey& nhg_key); + static inline unsigned getSyncedNhgCount() + { SWSS_LOG_ENTER(); return NhgBase::getSyncedCount(); } - /* Check if the group contains the given next hop. */ - inline bool hasNextHop(const NextHopKey& nh_key) const + /* Increase the number of synced next hop groups. */ + static void incSyncedNhgCount() { - return m_members.find(nh_key) != m_members.end(); - } - - /* Validate a next hop in the group, syncing it. */ - bool validateNextHop(const NextHopKey& nh_key); - - /* Invalidate a next hop in the group, removing it. */ - bool invalidateNextHop(const NextHopKey& nh_key); + SWSS_LOG_ENTER(); - /* Increment the number of existing groups. */ - static inline void incCount() { ++m_count; } + if (getSyncedNhgCount() >= m_maxNhgCount) + { + SWSS_LOG_ERROR("Incresing synced next hop group count beyond " + "switch's capabilities"); + throw logic_error("Next hop groups exceed switch's " + "capabilities"); + } - /* Decrement the number of existing groups. */ - static inline void decCount() { assert(m_count > 0); --m_count; } + NhgBase::incSyncedCount(); + } - /* Getters / Setters. */ - inline const NextHopGroupKey& getKey() const { return m_key; } - inline sai_object_id_t getId() const { return m_id; } - static inline unsigned int getCount() { return m_count; } - inline bool isTemp() const { return m_is_temp; } - inline bool isSynced() const { return m_id != SAI_NULL_OBJECT_ID; } - inline size_t getSize() const { return m_members.size(); } + /* Decrease the number of next hop groups. */ + static inline void decSyncedNhgCount() + { SWSS_LOG_ENTER(); NhgBase::decSyncedCount(); } - /* Convert NHG's details to a string. */ - std::string to_string() const + /* + * Check if the next hop group with the given index exists. + */ + inline bool hasNhg(const string &index) const { - return m_key.to_string() + ", SAI ID: " + std::to_string(m_id); + SWSS_LOG_ENTER(); + return nhgHandler.hasNhg(index) || cbfNhgHandler.hasNhg(index); } -private: - - /* The next hop group key of this group. */ - NextHopGroupKey m_key; - - /* The SAI ID of the group. */ - sai_object_id_t m_id; - - /* Members of this next hop group. */ - NhgMembers m_members; - - /* Whether the group is temporary or not. */ - bool m_is_temp; - /* - * Number of existing groups. Incremented when an object is created and - * decremented when an object is destroyed. This will also account for the - * groups created by RouteOrch. + * Get the next hop group with the given index. */ - static unsigned int m_count; - - /* Add group's members over the SAI API for the given keys. */ - bool syncMembers(const std::set& nh_keys); - - /* Remove group's members the SAI API from the given keys. */ - bool removeMembers(const std::set& nh_keys); - - /* Create the attributes vector for a next hop group member. */ - vector createNhgmAttrs(const NextHopGroupMember& nhgm) const; -}; - -/* - * Structure describing a next hop group which NhgOrch owns. Beside having a - * unique pointer to that next hop group, we also want to keep a ref count so - * NhgOrch knows how many other objects reference the next hop group in order - * not to remove them while still being referenced. - */ -struct NhgEntry -{ - /* Pointer to the next hop group. NhgOrch is the sole owner of it. */ - std::unique_ptr nhg; - - /* Number of external objects referencing this next hop group. */ - unsigned int ref_count; - - NhgEntry() = default; - explicit NhgEntry(std::unique_ptr&& _nhg, - unsigned int _ref_count = 0) : - nhg(std::move(_nhg)), ref_count(_ref_count) {} -}; - -/* - * Map indexed by next hop group's CP ID, containing the next hop group for - * that ID and the number of objects referencing it. - */ -typedef std::unordered_map NhgTable; + const NhgBase& getNhg(const string &index) const + { + SWSS_LOG_ENTER(); + + try + { + return nhgHandler.getNhg(index); + } + catch(const std::out_of_range &e) + { + return cbfNhgHandler.getNhg(index); + } + } -/* - * Next Hop Group Orchestrator class that handles NEXTHOP_GROUP_TABLE - * updates. - */ -class NhgOrch : public Orch -{ -public: /* - * Constructor. + * Increase the reference counter for the next hop group with the given + * index. */ - NhgOrch(DBConnector *db, string tableName); - - /* Check if the next hop group given by it's index exists. */ - inline bool hasNhg(const std::string& index) const + void incNhgRefCount(const string &index) { - return m_syncdNextHopGroups.find(index) != m_syncdNextHopGroups.end(); + SWSS_LOG_ENTER(); + + if (nhgHandler.hasNhg(index)) + { + nhgHandler.incNhgRefCount(index); + } + else + { + cbfNhgHandler.incNhgRefCount(index); + } } /* - * Get the next hop group given by it's index. If the index does not exist - * in map, a std::out_of_range exception will be thrown. + * Decrease the reference counter for the next hop group with the given + * index. */ - inline const NextHopGroup& getNhg(const std::string& index) const - { return *m_syncdNextHopGroups.at(index).nhg; } - - /* Add a temporary next hop group when resources are exhausted. */ - NextHopGroup createTempNhg(const NextHopGroupKey& nhg_key); - - /* Getters / Setters. */ - inline unsigned int getMaxNhgCount() const { return m_maxNhgCount; } - static inline unsigned int getNhgCount() { return NextHopGroup::getCount(); } - - /* Validate / Invalidate a next hop. */ - bool validateNextHop(const NextHopKey& nh_key); - bool invalidateNextHop(const NextHopKey& nh_key); - - /* Increase / Decrease the number of next hop groups. */ - inline void incNhgCount() + void decNhgRefCount(const string &index) { - assert(NextHopGroup::getCount() < m_maxNhgCount); - NextHopGroup::incCount(); + SWSS_LOG_ENTER(); + + if (nhgHandler.hasNhg(index)) + { + nhgHandler.decNhgRefCount(index); + } + else + { + cbfNhgHandler.decNhgRefCount(index); + } } - inline void decNhgCount() { NextHopGroup::decCount(); } - /* Increase / Decrease ref count for a NHG given by it's index. */ - void incNhgRefCount(const std::string& index); - void decNhgRefCount(const std::string& index); + void doTask(Consumer &consumer) override + { + SWSS_LOG_ENTER(); + + if (!gPortsOrch->allPortsReady()) + { + return; + } + + string table_name = consumer.getTableName(); + + if (table_name == APP_NEXTHOP_GROUP_TABLE_NAME) + { + nhgHandler.doTask(consumer); + } + else if (table_name == APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) + { + cbfNhgHandler.doTask(consumer); + } + } /* Handling SAI status*/ task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, void *context = nullptr) @@ -233,17 +142,15 @@ class NhgOrch : public Orch bool parseHandleSaiStatusFailure(task_process_status status) { return Orch::parseHandleSaiStatusFailure(status); } -private: - /* - * Switch's maximum number of next hop groups capacity. + * Handlers dealing with the (non) CBF operations. */ - unsigned int m_maxNhgCount; + NhgHandler nhgHandler; + CbfNhgHandler cbfNhgHandler; +private: /* - * The next hop group table. + * Switch's maximum number of next hop groups capacity. */ - NhgTable m_syncdNextHopGroups; - - void doTask(Consumer& consumer); + static unsigned m_maxNhgCount; }; diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index b990fa2a2a..3640756a8d 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -21,6 +21,8 @@ extern sai_switch_api_t* sai_switch_api; extern sai_object_id_t gSwitchId; extern bool gSaiRedisLogRotate; +unsigned NhgOrch::m_maxNhgCount = 0; + extern void syncd_apply_view(); /* * Global orch daemon variables @@ -32,6 +34,7 @@ IntfsOrch *gIntfsOrch; NeighOrch *gNeighOrch; RouteOrch *gRouteOrch; NhgOrch *gNhgOrch; +NhgMapOrch *gNhgMapOrch; FgNhgOrch *gFgNhgOrch; AclOrch *gAclOrch; PbhOrch *gPbhOrch; @@ -173,8 +176,8 @@ bool OrchDaemon::init() { APP_ROUTE_TABLE_NAME, routeorch_pri }, { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; + gNhgOrch = new NhgOrch(m_applDb, {APP_NEXTHOP_GROUP_TABLE_NAME, APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME}); gRouteOrch = new RouteOrch(m_applDb, route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch, gSrv6Orch); - gNhgOrch = new NhgOrch(m_applDb, APP_NEXTHOP_GROUP_TABLE_NAME); CoppOrch *copp_orch = new CoppOrch(m_applDb, APP_COPP_TABLE_NAME); TunnelDecapOrch *tunnel_decap_orch = new TunnelDecapOrch(m_applDb, APP_TUNNEL_DECAP_TABLE_NAME); @@ -202,7 +205,9 @@ bool OrchDaemon::init() CFG_WRED_PROFILE_TABLE_NAME, CFG_TC_TO_PRIORITY_GROUP_MAP_TABLE_NAME, CFG_PFC_PRIORITY_TO_PRIORITY_GROUP_MAP_TABLE_NAME, - CFG_PFC_PRIORITY_TO_QUEUE_MAP_TABLE_NAME + CFG_PFC_PRIORITY_TO_QUEUE_MAP_TABLE_NAME, + CFG_DSCP_TO_FC_MAP_TABLE_NAME, + CFG_EXP_TO_FC_MAP_TABLE_NAME }; QosOrch *qos_orch = new QosOrch(m_configDb, qos_tables); @@ -302,6 +307,8 @@ bool OrchDaemon::init() TableConnector stateDbBfdSessionTable(m_stateDb, STATE_BFD_SESSION_TABLE_NAME); gBfdOrch = new BfdOrch(m_applDb, APP_BFD_SESSION_TABLE_NAME, stateDbBfdSessionTable); + gNhgMapOrch = new NhgMapOrch(m_applDb, APP_FC_TO_NHG_INDEX_MAP_TABLE_NAME); + /* * The order of the orch list is important for state restore of warm start and * the queued processing in m_toSync map after gPortsOrch->allPortsReady() is set. @@ -310,7 +317,7 @@ bool OrchDaemon::init() * when iterating ConsumerMap. This is ensured implicitly by the order of keys in ordered map. * For cases when Orch has to process tables in specific order, like PortsOrch during warm start, it has to override Orch::doTask() */ - m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, mux_orch, mux_cb_orch, gIntfsOrch, gNeighOrch, gNhgOrch, gRouteOrch, copp_orch, qos_orch, wm_orch, policer_orch, tunnel_decap_orch, sflow_orch, debug_counter_orch, gMacsecOrch, gBfdOrch, gSrv6Orch}; + m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, mux_orch, mux_cb_orch, gIntfsOrch, gNeighOrch, gNhgMapOrch, gNhgOrch, gRouteOrch, copp_orch, qos_orch, wm_orch, policer_orch, tunnel_decap_orch, sflow_orch, debug_counter_orch, gMacsecOrch, gBfdOrch, gSrv6Orch}; bool initialize_dtel = false; if (platform == BFN_PLATFORM_SUBSTRING || platform == VS_PLATFORM_SUBSTRING) diff --git a/orchagent/orchdaemon.h b/orchagent/orchdaemon.h index 4be9b7eb9b..92527407a2 100644 --- a/orchagent/orchdaemon.h +++ b/orchagent/orchdaemon.h @@ -12,6 +12,7 @@ #include "neighorch.h" #include "routeorch.h" #include "nhgorch.h" +#include "nhgmaporch.h" #include "copporch.h" #include "tunneldecaporch.h" #include "qosorch.h" diff --git a/orchagent/qosorch.cpp b/orchagent/qosorch.cpp index 1430aceef1..fbb8c4a6b5 100644 --- a/orchagent/qosorch.cpp +++ b/orchagent/qosorch.cpp @@ -3,12 +3,14 @@ #include "logger.h" #include "crmorch.h" #include "sai_serialize.h" +#include "nhgmaporch.h" #include #include #include #include #include +#include using namespace std; @@ -51,7 +53,9 @@ map qos_to_attr_map = { {tc_to_pg_map_field_name, SAI_PORT_ATTR_QOS_TC_TO_PRIORITY_GROUP_MAP}, {pfc_to_pg_map_name, SAI_PORT_ATTR_QOS_PFC_PRIORITY_TO_PRIORITY_GROUP_MAP}, {pfc_to_queue_map_name, SAI_PORT_ATTR_QOS_PFC_PRIORITY_TO_QUEUE_MAP}, - {scheduler_field_name, SAI_PORT_ATTR_QOS_SCHEDULER_PROFILE_ID} + {scheduler_field_name, SAI_PORT_ATTR_QOS_SCHEDULER_PROFILE_ID}, + {dscp_to_fc_field_name, SAI_PORT_ATTR_QOS_DSCP_TO_FORWARDING_CLASS_MAP}, + {exp_to_fc_field_name, SAI_PORT_ATTR_QOS_MPLS_EXP_TO_FORWARDING_CLASS_MAP} }; map scheduler_meter_map = { @@ -70,7 +74,9 @@ type_map QosOrch::m_qos_maps = { {CFG_QUEUE_TABLE_NAME, new object_reference_map()}, {CFG_TC_TO_PRIORITY_GROUP_MAP_TABLE_NAME, new object_reference_map()}, {CFG_PFC_PRIORITY_TO_PRIORITY_GROUP_MAP_TABLE_NAME, new object_reference_map()}, - {CFG_PFC_PRIORITY_TO_QUEUE_MAP_TABLE_NAME, new object_reference_map()} + {CFG_PFC_PRIORITY_TO_QUEUE_MAP_TABLE_NAME, new object_reference_map()}, + {CFG_DSCP_TO_FC_MAP_TABLE_NAME, new object_reference_map()}, + {CFG_EXP_TO_FC_MAP_TABLE_NAME, new object_reference_map()}, }; map qos_to_ref_table_map = { @@ -82,9 +88,13 @@ map qos_to_ref_table_map = { {pfc_to_pg_map_name, CFG_PFC_PRIORITY_TO_PRIORITY_GROUP_MAP_TABLE_NAME}, {pfc_to_queue_map_name, CFG_PFC_PRIORITY_TO_QUEUE_MAP_TABLE_NAME}, {scheduler_field_name, CFG_SCHEDULER_TABLE_NAME}, - {wred_profile_field_name, CFG_WRED_PROFILE_TABLE_NAME} + {wred_profile_field_name, CFG_WRED_PROFILE_TABLE_NAME}, + {dscp_to_fc_field_name, CFG_DSCP_TO_FC_MAP_TABLE_NAME}, + {exp_to_fc_field_name, CFG_EXP_TO_FC_MAP_TABLE_NAME} }; +#define DSCP_MAX_VAL 63 +#define EXP_MAX_VAL 7 task_process_status QosMapHandler::processWorkItem(Consumer& consumer) { @@ -794,6 +804,180 @@ sai_object_id_t PfcToQueueHandler::addQosItem(const vector &att } +bool DscpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &tuple, vector &attributes) +{ + SWSS_LOG_ENTER(); + + sai_uint8_t max_fc_val = NhgMapOrch::getMaxFcVal(); + + sai_attribute_t list_attr; + list_attr.id = SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST; + list_attr.value.qosmap.count = (uint32_t)kfvFieldsValues(tuple).size(); + list_attr.value.qosmap.list = new sai_qos_map_t[list_attr.value.qosmap.count](); + uint32_t ind = 0; + + for (auto i = kfvFieldsValues(tuple).begin(); i != kfvFieldsValues(tuple).end(); i++, ind++) + { + try + { + auto value = stoi(fvField(*i)); + if (value < 0) + { + SWSS_LOG_ERROR("DSCP value %d is negative", value); + return false; + } + else if (value > DSCP_MAX_VAL) + { + SWSS_LOG_ERROR("DSCP value %d is greater than max value %d", value, DSCP_MAX_VAL); + return false; + } + list_attr.value.qosmap.list[ind].key.dscp = static_cast(value); + + value = stoi(fvValue(*i)); + if ((value < 0) || (value > max_fc_val)) + { + SWSS_LOG_ERROR("FC value %d is either negative, or bigger than max value %d", value, max_fc_val); + return false; + } + list_attr.value.qosmap.list[ind].value.fc = static_cast(value); + + SWSS_LOG_DEBUG("key.dscp:%d, value.fc:%d", + list_attr.value.qosmap.list[ind].key.dscp, + list_attr.value.qosmap.list[ind].value.fc); + } + catch(const invalid_argument& e) + { + SWSS_LOG_ERROR("Got exception during conversion: %s", e.what()); + return false; + } + } + attributes.push_back(list_attr); + return true; +} + +sai_object_id_t DscpToFcMapHandler::addQosItem(const vector &attributes) +{ + SWSS_LOG_ENTER(); + sai_status_t sai_status; + sai_object_id_t sai_object; + vector qos_map_attrs; + + sai_attribute_t qos_map_attr; + qos_map_attr.id = SAI_QOS_MAP_ATTR_TYPE; + qos_map_attr.value.u32 = SAI_QOS_MAP_TYPE_DSCP_TO_FORWARDING_CLASS; + qos_map_attrs.push_back(qos_map_attr); + + qos_map_attr.id = SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST; + qos_map_attr.value.qosmap.count = attributes[0].value.qosmap.count; + qos_map_attr.value.qosmap.list = attributes[0].value.qosmap.list; + qos_map_attrs.push_back(qos_map_attr); + + sai_status = sai_qos_map_api->create_qos_map(&sai_object, + gSwitchId, + (uint32_t)qos_map_attrs.size(), + qos_map_attrs.data()); + if (SAI_STATUS_SUCCESS != sai_status) + { + SWSS_LOG_ERROR("Failed to create dscp_to_fc map. status:%d", sai_status); + return SAI_NULL_OBJECT_ID; + } + SWSS_LOG_DEBUG("created QosMap object:%" PRIx64, sai_object); + return sai_object; +} + +task_process_status QosOrch::handleDscpToFcTable(Consumer& consumer) +{ + SWSS_LOG_ENTER(); + DscpToFcMapHandler dscp_fc_handler; + return dscp_fc_handler.processWorkItem(consumer); +} + +bool ExpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &tuple, + vector &attributes) +{ + SWSS_LOG_ENTER(); + + sai_uint8_t max_fc_val = NhgMapOrch::getMaxFcVal(); + + sai_attribute_t list_attr; + list_attr.id = SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST; + list_attr.value.qosmap.count = (uint32_t)kfvFieldsValues(tuple).size(); + list_attr.value.qosmap.list = new sai_qos_map_t[list_attr.value.qosmap.count](); + uint32_t ind = 0; + + for (auto i = kfvFieldsValues(tuple).begin(); i != kfvFieldsValues(tuple).end(); i++, ind++) + { + try + { + auto value = stoi(fvField(*i)); + if (value < 0) + { + SWSS_LOG_ERROR("EXP value %d is negative", value); + return false; + } + else if (value > EXP_MAX_VAL) + { + SWSS_LOG_ERROR("EXP value %d is greater than max value %d", value, EXP_MAX_VAL); + return false; + } + list_attr.value.qosmap.list[ind].key.mpls_exp = static_cast(value); + + value = stoi(fvValue(*i)); + if ((value < 0) || (value > max_fc_val)) + { + SWSS_LOG_ERROR("FC value %d is either negative, or bigger than max value %hu", value, max_fc_val); + return false; + } + list_attr.value.qosmap.list[ind].value.fc = static_cast(value); + + SWSS_LOG_DEBUG("key.mpls_exp:%d, value.fc:%d", + list_attr.value.qosmap.list[ind].key.mpls_exp, + list_attr.value.qosmap.list[ind].value.fc); + } + catch(const invalid_argument& e) + { + SWSS_LOG_ERROR("Got exception during conversion: %s", e.what()); + return false; + } + } + attributes.push_back(list_attr); + return true; +} + +sai_object_id_t ExpToFcMapHandler::addQosItem(const vector &attributes) +{ + SWSS_LOG_ENTER(); + sai_status_t sai_status; + sai_object_id_t sai_object; + vector qos_map_attrs; + + sai_attribute_t qos_map_attr; + qos_map_attr.id = SAI_QOS_MAP_ATTR_TYPE; + qos_map_attr.value.u32 = SAI_QOS_MAP_TYPE_MPLS_EXP_TO_FORWARDING_CLASS; + qos_map_attrs.push_back(qos_map_attr); + + qos_map_attr.id = SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST; + qos_map_attr.value.qosmap.count = attributes[0].value.qosmap.count; + qos_map_attr.value.qosmap.list = attributes[0].value.qosmap.list; + qos_map_attrs.push_back(qos_map_attr); + + sai_status = sai_qos_map_api->create_qos_map(&sai_object, gSwitchId, (uint32_t)qos_map_attrs.size(), qos_map_attrs.data()); + if (SAI_STATUS_SUCCESS != sai_status) + { + SWSS_LOG_ERROR("Failed to create exp_to_fc map. status:%d", sai_status); + return SAI_NULL_OBJECT_ID; + } + SWSS_LOG_DEBUG("created QosMap object:%" PRIx64, sai_object); + return sai_object; +} + +task_process_status QosOrch::handleExpToFcTable(Consumer& consumer) +{ + SWSS_LOG_ENTER(); + ExpToFcMapHandler exp_fc_handler; + return exp_fc_handler.processWorkItem(consumer); +} + task_process_status QosOrch::handlePfcToQueueTable(Consumer& consumer) { SWSS_LOG_ENTER(); @@ -825,6 +1009,8 @@ void QosOrch::initTableHandlers() m_qos_handler_map.insert(qos_handler_pair(CFG_QUEUE_TABLE_NAME, &QosOrch::handleQueueTable)); m_qos_handler_map.insert(qos_handler_pair(CFG_PORT_QOS_MAP_TABLE_NAME, &QosOrch::handlePortQosMapTable)); m_qos_handler_map.insert(qos_handler_pair(CFG_WRED_PROFILE_TABLE_NAME, &QosOrch::handleWredProfileTable)); + m_qos_handler_map.insert(qos_handler_pair(CFG_DSCP_TO_FC_MAP_TABLE_NAME, &QosOrch::handleDscpToFcTable)); + m_qos_handler_map.insert(qos_handler_pair(CFG_EXP_TO_FC_MAP_TABLE_NAME, &QosOrch::handleExpToFcTable)); m_qos_handler_map.insert(qos_handler_pair(CFG_TC_TO_PRIORITY_GROUP_MAP_TABLE_NAME, &QosOrch::handleTcToPgTable)); m_qos_handler_map.insert(qos_handler_pair(CFG_PFC_PRIORITY_TO_PRIORITY_GROUP_MAP_TABLE_NAME, &QosOrch::handlePfcPrioToPgTable)); diff --git a/orchagent/qosorch.h b/orchagent/qosorch.h index e65c3fa028..5e8f41e7c6 100644 --- a/orchagent/qosorch.h +++ b/orchagent/qosorch.h @@ -25,6 +25,8 @@ const string green_min_threshold_field_name = "green_min_threshold"; const string red_drop_probability_field_name = "red_drop_probability"; const string yellow_drop_probability_field_name = "yellow_drop_probability"; const string green_drop_probability_field_name = "green_drop_probability"; +const string dscp_to_fc_field_name = "dscp_to_fc_map"; +const string exp_to_fc_field_name = "exp_to_fc_map"; const string wred_profile_field_name = "wred_profile"; const string wred_red_enable_field_name = "wred_red_enable"; @@ -127,6 +129,20 @@ class PfcToQueueHandler : public QosMapHandler sai_object_id_t addQosItem(const vector &attributes); }; +class DscpToFcMapHandler : public QosMapHandler +{ +public: + bool convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &tuple, vector &attributes) override; + sai_object_id_t addQosItem(const vector &attributes) override; +}; + +class ExpToFcMapHandler : public QosMapHandler +{ +public: + bool convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &tuple, vector &attributes) override; + sai_object_id_t addQosItem(const vector &attributes) override; +}; + class QosOrch : public Orch { public: @@ -155,6 +171,8 @@ class QosOrch : public Orch task_process_status handleSchedulerTable(Consumer& consumer); task_process_status handleQueueTable(Consumer& consumer); task_process_status handleWredProfileTable(Consumer& consumer); + task_process_status handleDscpToFcTable(Consumer& consumer); + task_process_status handleExpToFcTable(Consumer& consumer); sai_object_id_t getSchedulerGroup(const Port &port, const sai_object_id_t queue_id); diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index 66c6dc5dc0..dd70891a10 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -761,8 +761,8 @@ void RouteOrch::doTask(Consumer& consumer) { try { - const NextHopGroup& nh_group = gNhgOrch->getNhg(nhg_index); - nhg = nh_group.getKey(); + const auto &nh_group = gNhgOrch->getNhg(nhg_index); + nhg = nh_group.getNhgKey(); ctx.using_temp_nhg = nh_group.isTemp(); } catch (const std::out_of_range& e) @@ -831,7 +831,7 @@ void RouteOrch::doTask(Consumer& consumer) // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (m_nextHopGroupCount + gNhgOrch->getNhgCount() >= m_maxNextHopGroupCount && + if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount && gRouteBulker.removing_entries_count() > 0) { break; @@ -1107,7 +1107,7 @@ bool RouteOrch::createFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id { SWSS_LOG_ENTER(); - if (m_nextHopGroupCount + gNhgOrch->getNhgCount() >= m_maxNextHopGroupCount) + if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1162,7 +1162,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) assert(!hasNextHopGroup(nexthops)); - if (m_nextHopGroupCount + gNhgOrch->getNhgCount() >= m_maxNextHopGroupCount) + if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1630,7 +1630,7 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) { try { - const NextHopGroup& nhg = gNhgOrch->getNhg(ctx.nhg_index); + const auto &nhg = gNhgOrch->getNhg(ctx.nhg_index); next_hop_id = nhg.getId(); } catch(const std::out_of_range& e) diff --git a/tests/mock_tests/Makefile.am b/tests/mock_tests/Makefile.am index f1d02898f9..934a172d19 100644 --- a/tests/mock_tests/Makefile.am +++ b/tests/mock_tests/Makefile.am @@ -39,7 +39,10 @@ tests_SOURCES = aclorch_ut.cpp \ $(top_srcdir)/orchagent/routeorch.cpp \ $(top_srcdir)/orchagent/mplsrouteorch.cpp \ $(top_srcdir)/orchagent/fgnhgorch.cpp \ - $(top_srcdir)/orchagent/nhgorch.cpp \ + $(top_srcdir)/orchagent/nhgbase.cpp \ + $(top_srcdir)/orchagent/nhghandler.cpp \ + $(top_srcdir)/orchagent/cbfnhghandler.cpp \ + $(top_srcdir)/orchagent/nhgmaporch.cpp \ $(top_srcdir)/orchagent/neighorch.cpp \ $(top_srcdir)/orchagent/intfsorch.cpp \ $(top_srcdir)/orchagent/portsorch.cpp \ diff --git a/tests/test_nhg.py b/tests/test_nhg.py index f1b600a22b..b57940021d 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -11,6 +11,7 @@ class TestNextHopGroupBase(object): ASIC_NHS_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" ASIC_NHG_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP" + ASIC_NHG_MAP_STR = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MAP" ASIC_NHGM_STR = ASIC_NHG_STR + "_MEMBER" ASIC_RT_STR = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" ASIC_INSEG_STR = "ASIC_STATE:SAI_OBJECT_TYPE_INSEG_ENTRY" @@ -74,6 +75,45 @@ def get_nhgm_ids(self, nhg_index): return nhgms + def get_nhg_map_id(self, nhg_map_index): + nhg_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_NEXTHOP_GROUP_TABLE_NAME) + cbf_nhg_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, + swsscommon.APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) + + asic_nhgs_count = len(self.asic_db.get_keys(self.ASIC_NHG_STR)) + + # Create a NHG + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), ('ifname', 'Ethernet0')]) + nhg_ps.set('testnhg', fvs) + + # Add a CBF NHG pointing to the given map + fvs = swsscommon.FieldValuePairs([('members', 'testnhg'), ('selection_map', nhg_map_index)]) + cbf_nhg_ps.set('testcbfnhg', fvs) + + # If the CBF NHG can't be created, the provided NHG map index is invalid + try: + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) + except: + # Remove the added NHGs + cbf_nhg_ps._del('testcbfnhg') + nhg_ps._del('testnhg') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + return None + + # Get the CBF NHG ID + cbf_nhg_id = self.get_nhg_id('testcbfnhg') + assert cbf_nhg_id != None + + # Get the NHG map id + nhg_map_id = self.asic_db.get_entry(self.ASIC_NHG_STR, cbf_nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] + + # Remove the added NHGs + cbf_nhg_ps._del('testcbfnhg') + nhg_ps._del('testnhg') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count) + + return nhg_map_id + def port_name(self, i): return "Ethernet" + str(i * 4) @@ -116,6 +156,11 @@ def init_test(self, dvs, num_intfs): self.nhg_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_NEXTHOP_GROUP_TABLE_NAME) self.rt_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_ROUTE_TABLE_NAME) self.lr_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_LABEL_ROUTE_TABLE_NAME) + self.cbf_nhg_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) + self.fc_to_nhg_ps = swsscommon.ProducerStateTable(self.app_db.db_connection, swsscommon.APP_FC_TO_NHG_INDEX_MAP_TABLE_NAME) + + # Set switch FC capability to 63 + self.dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') for i in range(num_intfs): self.config_intf(i) @@ -125,11 +170,18 @@ def init_test(self, dvs, num_intfs): self.asic_insgs_count = len(self.asic_db.get_keys(self.ASIC_INSEG_STR)) self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) self.asic_rts_count = len(self.asic_db.get_keys(self.ASIC_RT_STR)) + self.asic_nhg_maps_count = len(self.asic_db.get_keys(self.ASIC_NHG_MAP_STR)) def nhg_exists(self, nhg_index): return self.get_nhg_id(nhg_index) is not None -class TestNextHopGroupExhaust(TestNextHopGroupBase): + def route_exists(self, rt_prefix): + return self.get_route_id(rt_prefix) is not None + + def nhg_map_exists(self, nhg_map_index): + return self.get_nhg_map_id(nhg_map_index) is not None + +class TestNhgExhaustBase(TestNextHopGroupBase): MAX_ECMP_COUNT = 512 MAX_PORT_COUNT = 10 @@ -162,28 +214,200 @@ def gen_valid_binary(self): return binary gen_valid_binary.fmt = '{{0:0{}b}}'.format(MAX_PORT_COUNT) - def test_nhgorch_nhg_exhaust(self, dvs, testlog): - def gen_nhg_index(nhg_number): - return "group{}".format(nhg_number) - def create_temp_nhg(): +class TestRotueOrchNhgExhaust(TestNhgExhaustBase): + def test_route_nhg_exhaust(self, dvs, testlog): + """ + Test the situation of exhausting ECMP group, assume SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS is 512 + + In order to achieve that, we will config + 1. 9 ports + 2. 512 routes with different nexthop group + + See Also + -------- + SwitchStateBase::set_number_of_ecmp_groups() + https://github.com/Azure/sonic-sairedis/blob/master/vslib/src/SwitchStateBase.cpp + + """ + + # TODO: check ECMP 512 + + def gen_ipprefix(r): + """ Construct route like 2.X.X.0/24 """ + ip = ipaddress.IPv4Address(IP_INTEGER_BASE + r * 256) + ip = str(ip) + ipprefix = ip + "/24" + return ipprefix + + def asic_route_nhg_fvs(k): + fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) + if not fvs: + return None + + nhgid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") + if nhgid is None: + return None + + fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + return fvs + + if sys.version_info < (3, 0): + IP_INTEGER_BASE = int(ipaddress.IPv4Address(unicode("2.2.2.0"))) + else: + IP_INTEGER_BASE = int(ipaddress.IPv4Address(str("2.2.2.0"))) + + self.init_test(dvs) + + # Add first batch of routes with unique nexthop groups in AppDB + route_count = 0 + while route_count < self.MAX_ECMP_COUNT: + binary = self.gen_valid_binary() + fvs = self.gen_nhg_fvs(binary) + route_ipprefix = gen_ipprefix(route_count) + self.rt_ps.set(route_ipprefix, fvs) + route_count += 1 + + # Wait and check ASIC DB the count of nexthop groups used + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Wait and check ASIC DB the count of routes + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + self.MAX_ECMP_COUNT) + self.asic_rts_count += self.MAX_ECMP_COUNT + + # Add a route with labeled NHs + self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) + route_ipprefix = gen_ipprefix(route_count) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.rt_ps.set(route_ipprefix, fvs) + route_count += 1 + + # A temporary route should be created + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + + # A NH should be elected as the temporary NHG and it should be created + # as it doesn't exist. + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) + + # Delete the route. The route and the added labeled NH should be + # removed. + self.rt_ps._del(route_ipprefix) + route_count -= 1 + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) + + # Add second batch of routes with unique nexthop groups in AppDB + # Add more routes with new nexthop group in AppDBdd + route_ipprefix = gen_ipprefix(route_count) + base_ipprefix = route_ipprefix + base = route_count + route_count = 0 + while route_count < 10: + binary = self.gen_valid_binary() + fvs = self.gen_nhg_fvs(binary) + route_ipprefix = gen_ipprefix(base + route_count) + self.rt_ps.set(route_ipprefix, fvs) + route_count += 1 + last_ipprefix = route_ipprefix + + # Wait until we get expected routes and check ASIC DB on the count of nexthop groups used, and it should not increase + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 10) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + # Check the route points to next hop group + # Note: no need to wait here + k = self.get_route_id("2.2.2.0/24") + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert fvs is not None + + # Check the second batch does not point to next hop group + k = self.get_route_id(base_ipprefix) + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert not(fvs) + + # Remove first batch of routes with unique nexthop groups in AppDB + route_count = 0 + self.r = 0 + while route_count < self.MAX_ECMP_COUNT: + route_ipprefix = gen_ipprefix(route_count) + self.rt_ps._del(route_ipprefix) + route_count += 1 + self.asic_rts_count -= self.MAX_ECMP_COUNT + + # Wait and check the second batch points to next hop group + # Check ASIC DB on the count of nexthop groups used, and it should not increase or decrease + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 10) + k = self.get_route_id(base_ipprefix) + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert fvs is not None + k = self.get_route_id(last_ipprefix) + assert k is not None + fvs = asic_route_nhg_fvs(k) + assert fvs is not None + + # Cleanup + + # Remove second batch of routes + for i in range(10): + route_ipprefix = gen_ipprefix(self.MAX_ECMP_COUNT + i) + self.rt_ps._del(route_ipprefix) + + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 0) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + + +class TestNhgOrchNhgExhaust(TestNhgExhaustBase): + + def init_test(self, dvs): + super().init_test(dvs) + + self.nhg_count = self.asic_nhgs_count + self.first_valid_nhg = self.nhg_count + self.asic_nhgs = {} + + # Add first batch of next hop groups to reach the NHG limit + while self.nhg_count < self.MAX_ECMP_COUNT: binary = self.gen_valid_binary() nhg_fvs = self.gen_nhg_fvs(binary) - nhg_index = gen_nhg_index(self.nhg_count) + nhg_index = self.gen_nhg_index(self.nhg_count) self.nhg_ps.set(nhg_index, nhg_fvs) + + # Save the NHG index/ID pair + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + # Increase the number of NHGs in ASIC DB self.nhg_count += 1 + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + + def gen_nhg_index(self, nhg_number): + return "group{}".format(nhg_number) - return nhg_index, binary + def create_temp_nhg(self): + binary = self.gen_valid_binary() + nhg_fvs = self.gen_nhg_fvs(binary) + nhg_index = self.gen_nhg_index(self.nhg_count) + self.nhg_ps.set(nhg_index, nhg_fvs) + self.nhg_count += 1 - def delete_nhg(): - del_nhg_index = gen_nhg_index(self.first_valid_nhg) - del_nhg_id = self.asic_nhgs[del_nhg_index] + return nhg_index, binary - self.nhg_ps._del(del_nhg_index) - self.asic_nhgs.pop(del_nhg_index) - self.first_valid_nhg += 1 + def delete_nhg(self): + del_nhg_index = self.gen_nhg_index(self.first_valid_nhg) + del_nhg_id = self.asic_nhgs[del_nhg_index] - return del_nhg_id + self.nhg_ps._del(del_nhg_index) + self.asic_nhgs.pop(del_nhg_index) + self.first_valid_nhg += 1 + + return del_nhg_id + + + def test_nhgorch_nhg_exhaust(self, dvs, testlog): # Test scenario: # - create a NHG and assert a NHG object doesn't get added to ASIC DB @@ -191,7 +415,7 @@ def delete_nhg(): def temporary_group_promotion_test(): # Add a new next hop group - it should create a temporary one instead prev_nhgs = self.asic_db.get_keys(self.ASIC_NHG_STR) - nhg_index, _ = create_temp_nhg() + nhg_index, _ = self.create_temp_nhg() # Save the temporary NHG's SAI ID time.sleep(1) @@ -204,7 +428,7 @@ def temporary_group_promotion_test(): assert prev_nhgs == self.asic_db.get_keys(self.ASIC_NHG_STR) # Delete an existing next hop group - del_nhg_id = delete_nhg() + del_nhg_id = self.delete_nhg() # Wait for the key to be deleted self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) @@ -225,7 +449,7 @@ def group_update_test(): # Update a group binary = self.gen_valid_binary() nhg_fvs = self.gen_nhg_fvs(binary) - nhg_index = gen_nhg_index(self.first_valid_nhg) + nhg_index = self.gen_nhg_index(self.first_valid_nhg) # Save the previous members prev_nhg_members = self.get_nhgm_ids(nhg_index) @@ -241,7 +465,7 @@ def group_update_test(): # - create and delete a NHG while the ASIC DB is full and assert nothing changes def create_delete_temporary_test(): # Create a new temporary group - nhg_index, _ = create_temp_nhg() + nhg_index, _ = self.create_temp_nhg() time.sleep(1) # Delete the temporary group @@ -259,7 +483,7 @@ def create_delete_temporary_test(): # - delete a NHG and assert the new one is added and it has the updated number of members def update_temporary_group_test(): # Create a new temporary group - nhg_index, binary = create_temp_nhg() + nhg_index, binary = self.create_temp_nhg() # Save the number of group members binary_count = binary.count('1') @@ -275,7 +499,7 @@ def update_temporary_group_test(): self.nhg_ps.set(nhg_index, nhg_fvs) # Delete a group - del_nhg_id = delete_nhg() + del_nhg_id = self.delete_nhg() # Wait for the group to be deleted self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) @@ -296,7 +520,7 @@ def update_temporary_group_test(): # - delete a NHG and assert the temporary NHG is promoted and its SAI ID also changes def route_nhg_update_test(): # Add a route - nhg_index = gen_nhg_index(self.first_valid_nhg) + nhg_index = self.gen_nhg_index(self.first_valid_nhg) rt_fvs = swsscommon.FieldValuePairs([('nexthop_group', nhg_index)]) self.rt_ps.set('2.2.2.0/24', rt_fvs) @@ -307,7 +531,7 @@ def route_nhg_update_test(): prev_nhg_id = self.asic_nhgs[nhg_index] # Create a new temporary group - nhg_index, binary = create_temp_nhg() + nhg_index, binary = self.create_temp_nhg() # Get the route ID rt_id = self.get_route_id('2.2.2.0/24') @@ -349,7 +573,7 @@ def route_nhg_update_test(): {'SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID': prev_nhg_id}) # Delete a NHG. - del_nhg_id = delete_nhg() + del_nhg_id = self.delete_nhg() # Wait for the NHG to be deleted self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) @@ -376,7 +600,7 @@ def labeled_nhg_temporary_promotion_test(): fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), ('mpls_nh', 'push1,push3'), ('ifname', 'Ethernet0,Ethernet4')]) - nhg_index = gen_nhg_index(self.nhg_count) + nhg_index = self.gen_nhg_index(self.nhg_count) self.nhg_ps.set(nhg_index, fvs) self.nhg_count += 1 @@ -385,7 +609,7 @@ def labeled_nhg_temporary_promotion_test(): self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) # Delete a next hop group - delete_nhg() + self.delete_nhg() # The group should be promoted and the other labeled NH should also get # created @@ -409,7 +633,7 @@ def back_compatibility_test(): self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) # Delete a next hop group - del_nhg_id = delete_nhg() + del_nhg_id = self.delete_nhg() self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) # The temporary group should be promoted @@ -425,7 +649,7 @@ def invalid_temporary_test(): # Create a temporary NHG that contains only NHs that do not exist nhg_fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.21,10.0.0.23'), ('ifname', 'Ethernet40,Ethernet44')]) - nhg_index = gen_nhg_index(self.nhg_count) + nhg_index = self.gen_nhg_index(self.nhg_count) self.nhg_count += 1 self.nhg_ps.set(nhg_index, nhg_fvs) @@ -456,24 +680,6 @@ def invalid_temporary_test(): self.init_test(dvs) - self.nhg_count = self.asic_nhgs_count - self.first_valid_nhg = self.nhg_count - self.asic_nhgs = {} - - # Add first batch of next hop groups to reach the NHG limit - while self.nhg_count < self.MAX_ECMP_COUNT: - binary = self.gen_valid_binary() - nhg_fvs = self.gen_nhg_fvs(binary) - nhg_index = gen_nhg_index(self.nhg_count) - self.nhg_ps.set(nhg_index, nhg_fvs) - - # Save the NHG index/ID pair - self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) - - # Increase the number of NHGs in ASIC DB - self.nhg_count += 1 - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) - temporary_group_promotion_test() group_update_test() create_delete_temporary_test() @@ -495,149 +701,188 @@ def invalid_temporary_test(): self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) - def test_route_nhg_exhaust(self, dvs, testlog): - """ - Test the situation of exhausting ECMP group, assume SAI_SWITCH_ATTR_NUMBER_OF_ECMP_GROUPS is 512 - - In order to achieve that, we will config - 1. 9 ports - 2. 512 routes with different nexthop group - - See Also - -------- - SwitchStateBase::set_number_of_ecmp_groups() - https://github.com/Azure/sonic-sairedis/blob/master/vslib/src/SwitchStateBase.cpp - - """ - - # TODO: check ECMP 512 - - def gen_ipprefix(r): - """ Construct route like 2.X.X.0/24 """ - ip = ipaddress.IPv4Address(IP_INTEGER_BASE + r * 256) - ip = str(ip) - ipprefix = ip + "/24" - return ipprefix - - def asic_route_nhg_fvs(k): - fvs = self.asic_db.get_entry(self.ASIC_RT_STR, k) - if not fvs: - return None - - nhgid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") - if nhgid is None: - return None - - fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) - return fvs - - if sys.version_info < (3, 0): - IP_INTEGER_BASE = int(ipaddress.IPv4Address(unicode("2.2.2.0"))) - else: - IP_INTEGER_BASE = int(ipaddress.IPv4Address(str("2.2.2.0"))) - + def test_cbf_nhg_exhaust(self, dvs, testlog): self.init_test(dvs) - # Add first batch of routes with unique nexthop groups in AppDB - route_count = 0 - while route_count < self.MAX_ECMP_COUNT: - binary = self.gen_valid_binary() - fvs = self.gen_nhg_fvs(binary) - route_ipprefix = gen_ipprefix(route_count) - self.rt_ps.set(route_ipprefix, fvs) - route_count += 1 - - # Wait and check ASIC DB the count of nexthop groups used - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) - - # Wait and check ASIC DB the count of routes - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + self.MAX_ECMP_COUNT) - self.asic_rts_count += self.MAX_ECMP_COUNT + # Create an FC to NH index selection map + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 1) - # Add a route with labeled NHs - self.asic_nhs_count = len(self.asic_db.get_keys(self.ASIC_NHS_STR)) - route_ipprefix = gen_ipprefix(route_count) - fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push1,push3'), - ('ifname', 'Ethernet0,Ethernet4')]) - self.rt_ps.set(route_ipprefix, fvs) - route_count += 1 + # Create a CBF NHG group - it should fail + fvs = swsscommon.FieldValuePairs([('members', 'group2'), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + assert(not self.nhg_exists('cbfgroup1')) - # A temporary route should be created - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + # Delete a NHG + del_nhg_id = self.delete_nhg() + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) - # A NH should be elected as the temporary NHG and it should be created - # as it doesn't exist. - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) + # The CBF NHG should be created + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + assert(self.nhg_exists('cbfgroup1')) + nhg_id = self.get_nhg_id('cbfgroup1') - # Delete the route. The route and the added labeled NH should be - # removed. - self.rt_ps._del(route_ipprefix) - route_count -= 1 - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) - self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count) + # Create a temporary NHG + nhg_index, _ = self.create_temp_nhg() - # Add second batch of routes with unique nexthop groups in AppDB - # Add more routes with new nexthop group in AppDBdd - route_ipprefix = gen_ipprefix(route_count) - base_ipprefix = route_ipprefix - base = route_count - route_count = 0 - while route_count < 10: - binary = self.gen_valid_binary() - fvs = self.gen_nhg_fvs(binary) - route_ipprefix = gen_ipprefix(base + route_count) - self.rt_ps.set(route_ipprefix, fvs) - route_count += 1 - last_ipprefix = route_ipprefix + # Update the CBF NHG to contain the temporary NHG + fvs = swsscommon.FieldValuePairs([('members', nhg_index), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + nhgm_id = self.get_nhgm_ids('cbfgroup1')[0] + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + assert(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID'] != self.get_nhg_id('group2')) + + # Coverage testing: CbfNhgHandler::hasSameMembers() returns false + # + # Update the group while it has a temporary NHG to contain one more member. + # + # Update the group keeping the same number of members, but changing the fully fledged NHG with another one. + # + # Update the group reversing the order of the 2 members. + # + # Update the CBF NHG back to its original form to resume testing. + fvs = swsscommon.FieldValuePairs([('members', '{},group3'.format(nhg_index)), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + fvs = swsscommon.FieldValuePairs([('members', '{},group4'.format(nhg_index)), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + fvs = swsscommon.FieldValuePairs([('members', 'group4,{}'.format(nhg_index)), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + fvs = swsscommon.FieldValuePairs([('members', nhg_index), ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + + # Delete a NHG + nh_id = self.get_nhg_id(nhg_index) + del_nhg_id = self.delete_nhg() + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The NHG should be promoted and the CBF NHG member updated + nhgm_id = self.get_nhgm_ids('cbfgroup1')[0] + self.asic_db.wait_for_field_negative_match(self.ASIC_NHGM_STR, + nhgm_id, + {'SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID': nh_id}) + self.asic_nhgs[nhg_index] = self.get_nhg_id(nhg_index) + + # Create a couple more temporary NHGs + nhg_index1, _ = self.create_temp_nhg() + nhg_index2, _ = self.create_temp_nhg() + + # Update the CBF NHG to contain both of them as well + fvs = swsscommon.FieldValuePairs([('members', '{},{},{}'.format(nhg_index, nhg_index1, nhg_index2)), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + + # The previous CBF NHG member should be deleted + self.asic_db.wait_for_deleted_entry(self.ASIC_NHGM_STR, nhgm_id) + + # Create an FC to NH index selection map + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap2', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 2) + + # Update the selection map of the group + old_map = self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + members = {nhgm_id: self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) for nhgm_id in nhgm_ids} + fvs = swsscommon.FieldValuePairs([('members', '{},{},{}'.format(nhg_index, nhg_index1, nhg_index2)), + ('selection_map', 'cbfnhgmap2')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + + # Check that the map was updates, but the members weren't + self.asic_db.wait_for_field_negative_match(self.ASIC_NHG_STR, + nhg_id, + {'SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP': old_map}) + values = {} + for nhgm_id in nhgm_ids: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + assert(members[nhgm_id] == fvs) + values[nhgm_id] = fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID'] + + # Gradually delete temporary NHGs and check that the NHG members are + # updated + for i in range(2): + # Delete a temporary NHG + del_nhg_id = self.delete_nhg() + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + time.sleep(1) - # Wait until we get expected routes and check ASIC DB on the count of nexthop groups used, and it should not increase - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 10) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.MAX_ECMP_COUNT) + # Sleep 1 second to allow CBF NHG to update its member NH ID + time.sleep(1) - # Check the route points to next hop group - # Note: no need to wait here - k = self.get_route_id("2.2.2.0/24") - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert fvs is not None + # Check that one of the temporary NHGs has been updated + diff_count = 0 + for nhgm_id, nh_id in values.items(): + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + if nh_id != fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID']: + values[nhgm_id] = fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID'] + diff_count += 1 + assert(diff_count == 1) - # Check the second batch does not point to next hop group - k = self.get_route_id(base_ipprefix) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert not(fvs) + for index in [nhg_index1, nhg_index2]: + self.asic_nhgs[index] = self.get_nhg_id(index) - # Remove first batch of routes with unique nexthop groups in AppDB - route_count = 0 - self.r = 0 - while route_count < self.MAX_ECMP_COUNT: - route_ipprefix = gen_ipprefix(route_count) - self.rt_ps._del(route_ipprefix) - route_count += 1 - self.asic_rts_count -= self.MAX_ECMP_COUNT + # Create a temporary NHG + nhg_index, binary = self.create_temp_nhg() - # Wait and check the second batch points to next hop group - # Check ASIC DB on the count of nexthop groups used, and it should not increase or decrease - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 10) - k = self.get_route_id(base_ipprefix) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert fvs is not None - k = self.get_route_id(last_ipprefix) - assert k is not None - fvs = asic_route_nhg_fvs(k) - assert fvs is not None + # Update the CBF NHG to point to the new NHG + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + fvs = swsscommon.FieldValuePairs([('members', nhg_index), ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) - # Cleanup + # Update the temporary NHG to force using a new designated NH + new_binary = [] - # Remove second batch of routes - for i in range(10): - route_ipprefix = gen_ipprefix(self.MAX_ECMP_COUNT + i) - self.rt_ps._del(route_ipprefix) + for i in range(len(binary)): + if binary[i] == '1': + new_binary.append('0') + else: + new_binary.append('1') + + binary = ''.join(new_binary) + assert binary.count('1') > 1 + + nh_id = self.get_nhg_id(nhg_index) + nhg_fvs = self.gen_nhg_fvs(binary) + self.nhg_ps.set(nhg_index, nhg_fvs) + + # The CBF NHG member's NH attribute should be updated + nhgm_id = self.get_nhgm_ids('cbfgroup1')[0] + self.asic_db.wait_for_field_negative_match(self.ASIC_NHGM_STR, + nhgm_id, + {'SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID': nh_id}) + + # Delete a NHG + del_nhg_id = self.delete_nhg() + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_STR, del_nhg_id) + + # The temporary group should be promoted + nh_id = self.get_nhg_id(nhg_index) + self.asic_db.wait_for_field_match(self.ASIC_NHGM_STR, + nhgm_id, + {'SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID': nh_id}) + self.asic_nhgs[nhg_index] = nh_id + + # Delete the NHGs + self.cbf_nhg_ps._del('cbfgroup1') + for k in self.asic_nhgs: + self.nhg_ps._del(k) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) - self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, 0) - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + # Delete the NHG maps + self.fc_to_nhg_ps._del('cbfnhgmap1') + self.fc_to_nhg_ps._del('cbfnhgmap2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count) class TestNextHopGroup(TestNextHopGroupBase): @@ -956,9 +1201,9 @@ def mainline_labeled_nhs_test(): def routeorch_nhgorch_interop_test(): # Create a route with labeled NHs fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), - ('mpls_nh', 'push1,push3'), - ('ifname', 'Ethernet0,Ethernet4'), - ('weight', '2,4')]) + ('mpls_nh', 'push1,push3'), + ('ifname', 'Ethernet0,Ethernet4'), + ('weight', '2,4')]) self.rt_ps.set('2.2.2.0/24', fvs) self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) @@ -1379,6 +1624,495 @@ def test_nhgorch_label_route(self, dvs, testlog): self.nhg_ps._del("group1") self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) +class TestCbfNextHopGroup(TestNextHopGroupBase): + MAX_NHG_MAP_COUNT = 512 + + def test_cbf_nhgorch(self, dvs, testlog): + + # Test scenario: + # - create two NHGs and a NHG map + # - create a CBF NHG and assert it has the expected details + # - delete the CBF NHG + def mainline_cbf_nhg_test(): + # Create two non-CBF NHGs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + self.nhg_ps.set("group1", fvs) + + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set('group2', fvs) + + # Wait for the groups to appear in ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + + # Create an FC to NH index selection map + nhg_map = [(str(i), '0' if i < 4 else '1') for i in range(8)] + fvs = swsscommon.FieldValuePairs(nhg_map) + self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 1) + + # Create a CBF NHG + fvs = swsscommon.FieldValuePairs([('members', 'group1,group2'), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + + # Check if the group makes it to ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + nhgid = self.get_nhg_id('cbfgroup1') + assert(nhgid != None) + fvs = self.asic_db.get_entry(self.ASIC_NHG_STR, nhgid) + assert(fvs['SAI_NEXT_HOP_GROUP_ATTR_TYPE'] == 'SAI_NEXT_HOP_GROUP_TYPE_CLASS_BASED') + + # Check if the next hop group members get created + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 7) + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + assert(len(nhgm_ids) == 2) + nhg_ids = dict(zip([self.get_nhg_id(x) for x in ['group1', 'group2']], ['0', '1'])) + for nhgm_id in nhgm_ids: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + nh_id = fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID'] + assert(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX'] == nhg_ids[nh_id]) + nhg_ids.pop(nh_id) + + # Delete the CBF next hop group + self.cbf_nhg_ps._del('cbfgroup1') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + + # Test scenario: + # - try creating invalid NHG maps and assert they're not being created + def data_validation_test(): + # Test the data validation + fv_values = [ + ('', 'cbfnhgmap1'), # empty members + ('group1,group2', ''), # non-existent selection map + ('group1,group1', 'cbfnhgmap1'), # non-unique members + ] + + for members, selection_map in fv_values: + fvs = swsscommon.FieldValuePairs([('members', members), + ('selection_map', selection_map)]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + assert(len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + 2) + + # Test scenario: + # - create a CBF NHG + # - try deleting one of its members and assert it fails as it is being referenced + # - update the group replacing the to-be-deleted member with another NHG and assert it is deleted + # - update the CBF NHG reordering the members and assert the new details match + def update_cbf_nhg_members_test(): + # Create a NHG with a single next hop + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1'), + ("ifname", "Ethernet0")]) + self.nhg_ps.set("group3", fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + + # Create a CBF NHG + fvs = swsscommon.FieldValuePairs([('members', 'group1,group3'), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 4) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 8) + + # Remove group1 while being referenced - should not work + self.nhg_ps._del('group1') + time.sleep(1) + assert(self.nhg_exists('group1')) + + # Update the CBF NHG replacing group1 with group2 + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + fvs = swsscommon.FieldValuePairs([('members', 'group2,group3'), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) + # group1 should be deleted and the updated members added + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 5) + + # Update the CBF NHG changing the order of the members + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + fvs = swsscommon.FieldValuePairs([('members', 'group3,group2'), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 5) + assert(len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + 3) + + indexes = { + self.get_nhg_id('group3'): '0', + self.get_nhg_id('group2'): '1' + } + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + for nhgm_id in nhgm_ids: + fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) + nh_id = fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID'] + assert(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX'] == indexes[nh_id]) + indexes.pop(nh_id) + + # Test scenario: + # - update the selection map of the CBF NHG and assert it changes in ASIC DB + def update_cbf_nhg_map_test(): + # Create an FC to NH index selection map + fvs = swsscommon.FieldValuePairs([('0', '1'), ('1', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap2', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 2) + + # Update the CBF NHG class map + nhg_id = self.get_nhg_id('cbfgroup1') + old_map = self.get_nhg_map_id('cbfnhgmap1') + fvs = swsscommon.FieldValuePairs([('members', 'group3,group2'), + ('selection_map', 'cbfnhgmap2')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_field_negative_match(self.ASIC_NHG_STR, + nhg_id, + {'SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP': old_map}) + assert(len(self.asic_db.get_keys(self.ASIC_NHG_STR)) == self.asic_nhgs_count + 3) + + # Test scenario: + # - create a route pointing to the CBF NHG + # - try deleting the CBF NHG and assert it fails as it is being referenced + # - delete the route and assert the CBF NHG also gets deleted + def delete_referenced_cbf_nhg_test(): + # Create a route pointing to the CBF NHG + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'cbfgroup1')]) + self.rt_ps.set('2.2.2.0/24', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + + # Try deleting the CBF NHG - should not work + self.cbf_nhg_ps._del('cbfgroup1') + time.sleep(1) + assert(self.nhg_exists('cbfgroup1')) + + # Delete the route - the CBF NHG should also get deleted + self.rt_ps._del('2.2.2.0/24') + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) + + # Test scenario: + # - create a route pointing to a CBF NHG that doesn't exist and assert it isn't created + # - create the CBF NHG and assert the route also gets created + def create_route_inexistent_cbf_nhg_test(): + # Create a route pointing to a CBF NHG that does not yet exist + fvs = swsscommon.FieldValuePairs([('nexthop_group', 'cbfgroup1')]) + self.rt_ps.set('2.2.2.0/24', fvs) + time.sleep(1) + assert(not self.route_exists('2.2.2.0/24')) + + # Create the CBF NHG + fvs = swsscommon.FieldValuePairs([('members', 'group3,group2'), + ('selection_map', 'cbfnhgmap2')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 5) + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + + # Test scenario: + # - try deleting the CBF NHG and assert it fails as it is being referenced + # - update the CBF NHG + # - delete the route pointing to the CBF NHG and assert the CBF NHG doesn't get deleted + def update_deleting_cbf_nhg_test(): + # Try deleting the CBF group + self.cbf_nhg_ps._del('cbfgroup1') + time.sleep(1) + assert(self.nhg_exists('cbfgroup1')) + + # Update the CBF group + nhgm_ids = self.get_nhgm_ids('cbfgroup1') + nhg_id = self.get_nhg_id('cbfgroup1') + old_map = self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] + fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) + nhgm_id = self.get_nhgm_ids('cbfgroup1')[0] + self.asic_db.wait_for_field_match(self.ASIC_NHGM_STR, + nhgm_id, + {'SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX': '0'}) + self.asic_db.wait_for_field_negative_match(self.ASIC_NHG_STR, + nhg_id, + {'SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP': old_map}) + + # Delete the route + self.rt_ps._del('2.2.2.0/24') + self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + # The update should have overwritten the delete for CBF NHG + assert(self.nhg_exists('cbfgroup1')) + + # Test scenario: + # - try updating the CBF NHG with a member that doesn't exist and assert the CBF NHG's + # - create the missing NHG and assert the CBF NHG's member also gets created + def update_cbf_nhg_inexistent_member_test(): + # Update the CBF NHG referencing an NHG that doesn't exist. In the end, create the NHG and + # make sure everything works fine. + fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + assert(len(self.asic_db.get_keys(self.ASIC_NHGM_STR)) == self.asic_nhgms_count) + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set('group2', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) + + # Test scenario: + # - try updating the CBF NHG with a NHG map that doesn't exist and assert it does't change + # - create the missing NHG map and assert the selection map also gets updated + def update_cbf_nhg_inexistent_map_test(): + # Update the CBF NHG with an NHG map that doesn't exist, then update it with one that does exist. + nhg_id = self.get_nhg_id('cbfgroup1') + smap_id = self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] + fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'a')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + assert(self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] == smap_id) + fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap2')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_field_negative_match(self.ASIC_NHG_STR, + nhg_id, + {'SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP': smap_id}) + + self.init_test(dvs, 4) + + mainline_cbf_nhg_test() + data_validation_test() + update_cbf_nhg_members_test() + update_cbf_nhg_map_test() + delete_referenced_cbf_nhg_test() + create_route_inexistent_cbf_nhg_test() + update_deleting_cbf_nhg_test() + + # Delete the NHGs + self.cbf_nhg_ps._del('cbfgroup1') + self.nhg_ps._del('group2') + self.nhg_ps._del('group3') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count) + + update_cbf_nhg_inexistent_member_test() + update_cbf_nhg_inexistent_map_test() + + # Coverage testing: Update the CBF NHG with a member that does not exist. + fvs = swsscommon.FieldValuePairs([('members', 'group3'), ('selection_map', 'cbfnhgmap2')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + time.sleep(1) + + # Delete the NHGs + self.cbf_nhg_ps._del('cbfgroup1') + self.nhg_ps._del('group2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + + # Delete the NHG maps + self.fc_to_nhg_ps._del('cbfnhgmap1') + self.fc_to_nhg_ps._del('cbfnhgmap2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count) + + # Coverage testing: Delete inexistent CBF NHG + self.cbf_nhg_ps._del('cbfgroup1') + + def test_nhg_map(self, dvs, testlog): + + # Test scenario: + # - create a NHG map and assert the expected details + # - delete the map + def mainline_nhg_map_test(): + # Create an FC to NH index map + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 1) + + # Check the NHG map attributes + nhg_map_id = self.get_nhg_map_id('cbfnhgmap1') + assert nhg_map_id != None + + self.asic_db.wait_for_field_match(self.ASIC_NHG_MAP_STR, + nhg_map_id, + {'SAI_NEXT_HOP_GROUP_MAP_ATTR_TYPE': 'SAI_NEXT_HOP_GROUP_MAP_TYPE_FORWARDING_CLASS_TO_INDEX', + 'SAI_NEXT_HOP_GROUP_MAP_ATTR_MAP_TO_VALUE_LIST': '{\"count\":1,\"list\":[{\"key\":0,\"value\":0}]}'}) + + # Delete the map + self.fc_to_nhg_ps._del('cbfnhgmap1') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count) + + # Test scenario: + # - try creating different NHG maps with invalid data and assert they don't get created + def data_validation_test(): + # Test validation errors + nhg_maps = [ + ('-1', '0'), # negative FC + ('64', '0'), # greater than max FC value + ('a', '0'), # non-integer FC + ('0', '-1'), # negative NH index + ('0', 'a'), # non-integer NH index + ] + + # Check that no such NHG map gets created + for nhg_map in nhg_maps: + fvs = swsscommon.FieldValuePairs([nhg_map]) + self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) + time.sleep(1) + assert(not self.nhg_map_exists('cbfnhgmap1')) + + # Test scenario: + # - create two CBF NHG and a NHG map + # - try deleting the NHG map and assert it fails as it is being referenced + # - delete a CBF NHG and assert the NHG map is still not deleted as it's still being referenced + # - create a new NHG map and update the CBF NHG to point to it and assert the previous NHG map gets deleted + def delete_referenced_nhg_map_test(): + # Create two non-CBF NHGs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + self.nhg_ps.set("group1", fvs) + + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set('group2', fvs) + + # Wait for the groups to appear in ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + + # Create an FC to NH index map + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 1) + + # Create a couple of CBF NHGs + for i in range(2): + fvs = swsscommon.FieldValuePairs([('members', 'group{}'.format(i + 1)), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup{}'.format(i), fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 4) + # Try deleting the NHG map. It should fail as it is referenced. + self.fc_to_nhg_ps._del('cbfnhgmap1') + time.sleep(1) + assert(self.nhg_map_exists('cbfnhgmap1')) + + # Delete a CBF NHG + self.cbf_nhg_ps._del('cbfgroup1') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + + # The NHG map is still referenced so shouldn't be deleted + assert(self.nhg_map_exists('cbfnhgmap1')) + + # Create a new FC to NH index map + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap2', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 2) + + # Update the second CBF NHG to point to the new map + fvs = swsscommon.FieldValuePairs([('members', 'group1'), ('selection_map', 'cbfnhgmap2')]) + self.cbf_nhg_ps.set('cbfgroup0', fvs) + + # The NHG map is no longer referenced and should have been deleted + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 1) + + # Test scenario: + # - delete the NHG map and assert it fails as it's being referenced + # - update the NHG map and assert for the new details + # - delete the referencing CBF NHG and assert the NHG map isn't deleted as it was updated + # - delete the NHG map + def update_override_delete_test(): + # Delete the second NHG map. It fails as it is referenced. + self.fc_to_nhg_ps._del('cbfnhgmap2') + time.sleep(1) + assert(self.nhg_map_exists('cbfnhgmap2')) + + # Update the NHG map + nhg_map_id = self.get_nhg_map_id('cbfnhgmap2') + fvs = swsscommon.FieldValuePairs([('1', '0'), ('2', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap2', fvs) + self.asic_db.wait_for_field_match(self.ASIC_NHG_MAP_STR, + nhg_map_id, + {'SAI_NEXT_HOP_GROUP_MAP_ATTR_TYPE': 'SAI_NEXT_HOP_GROUP_MAP_TYPE_FORWARDING_CLASS_TO_INDEX', + 'SAI_NEXT_HOP_GROUP_MAP_ATTR_MAP_TO_VALUE_LIST': '{\"count\":2,\"list\":[{\"key\":2,\"value\":0},{\"key\":1,\"value\":0}]}'}) + + # Delete the second CBF NHG + self.cbf_nhg_ps._del('cbfgroup0') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + + # The NHG map should not be deleted as it was updated + time.sleep(1) + assert(self.nhg_map_exists('cbfnhgmap2')) + + # Delete the NHG map + self.fc_to_nhg_ps._del('cbfnhgmap2') + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count) + + # Test scenario: + # - fill the ASIC with NHG maps + # - try creating a new NHG map and assert it fails as the ASIC is full + # - delete a NHG map and assert the to-be-created one is actually created + def nhg_map_exhaust_test(): + # Create the maximum number of NHG maps allowed by the VS switch (512) + fvs = swsscommon.FieldValuePairs([('0', '0')]) + for i in range(512 - self.asic_nhg_maps_count): + nhg_maps.append('cbfnhgmap{}'.format(i)) + self.fc_to_nhg_ps.set('cbfnhgmap{}'.format(i), fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.MAX_NHG_MAP_COUNT) + + # Try creating a new NHG map. It should fail as there is no more space. + nhg_maps.append('cbfnhgmap512') + self.fc_to_nhg_ps.set('cbfnhgmap512', fvs) + time.sleep(1) + assert(not self.nhg_map_exists('cbfnhgmap512')) + + # Delete an existing NHG map. The pending NHG map should now be created + del_nhg_map_index = nhg_maps.pop(0) + del_nhg_map_id = self.get_nhg_map_id(del_nhg_map_index) + self.fc_to_nhg_ps._del(del_nhg_map_index) + self.asic_db.wait_for_deleted_entry(self.ASIC_NHG_MAP_STR, del_nhg_map_id) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.MAX_NHG_MAP_COUNT) + assert(self.nhg_map_exists('cbfnhgmap512')) + + # Test scenario: + # - create a NHG map while the ASIC is full + # - create a CBF NHG pointing to this NHG map and assert it isn't created + # - delete a NHG map and assert the new NHG map and the CBF NHG map is created + # - delete the CBF NHG + def create_cbf_nhg_inexistent_map_test(): + # Create a new NHG map. It should remain pending. + nhg_maps.append('cbfnhgmap513') + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap513', fvs) + + # Create a CBF NHG which should reference this NHG map. It should fail creating it. + fvs = swsscommon.FieldValuePairs([('members', 'group1'), ('selection_map', 'cbfnhgmap513')]) + self.cbf_nhg_ps.set('testcbfnhg', fvs) + time.sleep(1) + assert(not self.nhg_exists('testcbfnhg')) + + # Delete an existing NHG map. The new map and the CBF NHG should be created. + self.fc_to_nhg_ps._del(nhg_maps.pop(0)) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.MAX_NHG_MAP_COUNT) + + self.cbf_nhg_ps._del('testcbfnhg') + + self.init_test(dvs, 4) + nhg_maps = [] + + mainline_nhg_map_test() + data_validation_test() + delete_referenced_nhg_map_test() + update_override_delete_test() + nhg_map_exhaust_test() + create_cbf_nhg_inexistent_map_test() + + # Cleanup + + # Delete the NHGs + for i in range(2): + self.nhg_ps._del('group{}'.format(i + 1)) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + + # Delete all the NHG maps + while len(nhg_maps) > 0: + self.fc_to_nhg_ps._del(nhg_maps.pop()) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count) + + # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying def test_nonflaky_dummy(): diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index 51304d8b1f..b51d5cbb49 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -47,7 +47,6 @@ def create_dot1p_profile(self): tbl.set(CFG_DOT1P_TO_TC_MAP_KEY, fvs) time.sleep(1) - def find_dot1p_profile(self): found = False dot1p_tc_map_raw = None @@ -86,7 +85,7 @@ def test_dot1p_cfg(self, dvs): self.create_dot1p_profile() _, dot1p_tc_map_raw = self.find_dot1p_profile() - dot1p_tc_map = json.loads(dot1p_tc_map_raw); + dot1p_tc_map = json.loads(dot1p_tc_map_raw) for dot1p2tc in dot1p_tc_map['list']: dot1p = str(dot1p2tc['key']['dot1p']) tc = str(dot1p2tc['value']['tc']) @@ -115,6 +114,158 @@ def test_port_dot1p(self, dvs): port_cnt = len(swsscommon.Table(self.config_db, CFG_PORT_TABLE_NAME).getKeys()) assert port_cnt == cnt +class TestCbf(object): + ASIC_QOS_MAP_STR = "ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP" + ASIC_PORT_STR = "ASIC_STATE:SAI_OBJECT_TYPE_PORT" + + def init_test(self, dvs): + self.dvs = dvs + self.asic_db = dvs.get_asic_db() + self.config_db = dvs.get_config_db() + self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) + self.asic_qos_map_count = len(self.asic_qos_map_ids) + self.dscp_ps = swsscommon.Table(self.config_db.db_connection, swsscommon.CFG_DSCP_TO_FC_MAP_TABLE_NAME) + self.exp_ps = swsscommon.Table(self.config_db.db_connection, swsscommon.CFG_EXP_TO_FC_MAP_TABLE_NAME) + + # Set switch FC capability to 63 + dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') + + def get_qos_id(self): + diff = set(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) - set(self.asic_qos_map_ids) + assert len(diff) <= 1 + return None if len(diff) == 0 else diff.pop() + + def test_dscp_to_fc(self, dvs): + self.init_test(dvs) + + # Create a DSCP_TO_FC map + dscp_map = [(str(i), str(i)) for i in range(0, 64)] + self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) + + self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) + + # Get the DSCP map ID + dscp_map_id = self.get_qos_id() + assert(dscp_map_id is not None) + + # Assert the expected values + fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) + assert(fvs.get("SAI_QOS_MAP_ATTR_TYPE") == "SAI_QOS_MAP_TYPE_DSCP_TO_FORWARDING_CLASS") + + # Apply port configuration + # self.port_qos_map_profile_test('dscp_to_fc_map', swsscommon.CFG_DSCP_TO_FC_MAP_TABLE_NAME, 'SAI_PORT_ATTR_QOS_DSCP_TO_FORWARDING_CLASS_MAP') + + # Delete the map + self.dscp_ps._del("AZURE") + self.asic_db.wait_for_deleted_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) + + # Test validation + maps = [ + ('-1', '0'), # negative DSCP + ('64', '0'), # DSCP greater than max value + ('0', '-1'), # negative FC + ('0', '64'), # FC greater than max value + ('a', '0'), # non-integer DSCP + ('0', 'a'), # non-integet FC + ] + + for fvs in maps: + dscp_ps.set('AZURE', swsscommon.FieldValuePairs([fvs])) + time.sleep(1) + assert(asic_qos_map_count == len(asic_db.get_keys('ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP'))) + dscp_ps._del("AZURE") + + # Delete a map that does not exist. Nothing should happen + self.dscp_ps._del("AZURE") + time.sleep(1) + assert(len(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) == self.asic_qos_map_count) + + def test_exp_to_fc(self, dvs): + self.init_test(dvs) + + # Create a EXP_TO_FC map + exp_map = [(str(i), str(i)) for i in range(0, 8)] + self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) + + self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) + + # Get the EXP map ID + exp_map_id = self.get_qos_id() + + # Assert the expected values + fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, exp_map_id) + assert(fvs.get("SAI_QOS_MAP_ATTR_TYPE") == "SAI_QOS_MAP_TYPE_MPLS_EXP_TO_FORWARDING_CLASS") + + # Delete the map + self.exp_ps._del("AZURE") + self.asic_db.wait_for_deleted_entry(self.ASIC_QOS_MAP_STR, exp_map_id) + + # Test validation + maps = [ + ('-1', '0'), # negative EXP + ('8', '0'), # EXP greater than max value + ('0', '-1'), # negative FC + ('0', '64'), # FC greater than max value + ('a', '0'), # non-integer EXP + ('0', 'a'), # non-integet FC + ] + + for fvs in maps: + exp_ps.set('AZURE', swsscommon.FieldValuePairs([fvs])) + time.sleep(1) + assert(asic_qos_map_count == len(asic_db.get_keys('ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP'))) + exp_ps._del("AZURE") + + # Update the map with valid values + exp_map = [(str(i), str(i + 10)) for i in range(0, 8)] + self.exp_ps.set('AZURE', swsscommon.FieldValuePairs(exp_map)) + self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) + + # Delete the map + exp_map_id = self.get_qos_id() + self.exp_ps._del("AZURE") + self.asic_db.wait_for_deleted_entry(self.ASIC_QOS_MAP_STR, exp_map_id) + + # Delete a map that does not exist. Nothing should happen + self.exp_ps._del("AZURE") + time.sleep(1) + assert(len(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) == self.asic_qos_map_count) + + def test_per_port_cbf_binding(self, dvs): + self.init_test(dvs) + + # Create a DSCP_TO_FC map + dscp_map = [(str(i), str(i)) for i in range(0, 64)] + self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) + self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) + dscp_map_id = self.get_qos_id() + self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) + + # Create a EXP_TO_FC map + exp_map = [(str(i), str(i)) for i in range(0, 8)] + self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) + self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 2) + exp_map_id = self.get_qos_id() + + tbl = swsscommon.Table(self.config_db.db_connection, swsscommon.CFG_PORT_QOS_MAP_TABLE_NAME) + fvs = swsscommon.FieldValuePairs([('dscp_to_fc_map', "AZURE"), + ('exp_to_fc_map', "AZURE")]) + keys = self.config_db.get_keys(swsscommon.CFG_PORT_TABLE_NAME) + for key in keys: + tbl.set(key, fvs) + time.sleep(1) + + dscp_cnt = 0 + exp_cnt = 0 + for key in self.asic_db.get_keys(self.ASIC_PORT_STR): + fvs = self.asic_db.get_entry(self.ASIC_PORT_STR, key) + if 'SAI_PORT_ATTR_QOS_DSCP_TO_FORWARDING_CLASS_MAP' in fvs: + assert fvs['SAI_PORT_ATTR_QOS_DSCP_TO_FORWARDING_CLASS_MAP'] == dscp_map_id + dscp_cnt += 1 + if 'SAI_PORT_ATTR_QOS_MPLS_EXP_TO_FORWARDING_CLASS_MAP' in fvs: + assert fvs['SAI_PORT_ATTR_QOS_MPLS_EXP_TO_FORWARDING_CLASS_MAP'] == exp_map_id + exp_cnt += 1 + assert dscp_cnt == exp_cnt == len(keys) class TestMplsTc(object): def connect_dbs(self, dvs): From 359a2e7c35fa21c274b4b9427af32c4d4cbe1aaa Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 19 Oct 2021 09:35:26 +0100 Subject: [PATCH 02/29] Restructure code --- orchagent/Makefile.am | 10 +- orchagent/{ => nhg}/cbfnhghandler.cpp | 22 ++--- orchagent/{ => nhg}/cbfnhghandler.h | 0 orchagent/{ => nhg}/nhgbase.cpp | 15 ++- orchagent/{ => nhg}/nhgbase.h | 64 ++++--------- orchagent/{ => nhg}/nhghandler.cpp | 69 +++++++------- orchagent/{ => nhg}/nhghandler.h | 0 orchagent/nhgmaporch.cpp | 2 - orchagent/nhgorch.cpp | 79 ++++++++++++++++ orchagent/nhgorch.h | 128 +++++--------------------- orchagent/orchdaemon.cpp | 4 +- tests/mock_tests/Makefile.am | 7 +- 12 files changed, 190 insertions(+), 210 deletions(-) rename orchagent/{ => nhg}/cbfnhghandler.cpp (96%) rename orchagent/{ => nhg}/cbfnhghandler.h (100%) rename orchagent/{ => nhg}/nhgbase.cpp (70%) rename orchagent/{ => nhg}/nhgbase.h (84%) rename orchagent/{ => nhg}/nhghandler.cpp (93%) rename orchagent/{ => nhg}/nhghandler.h (100%) create mode 100644 orchagent/nhgorch.cpp diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index 07007bbe52..ed30d16434 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -3,7 +3,8 @@ INCLUDES = -I $(top_srcdir)/lib \ -I $(top_srcdir)/warmrestart \ -I flex_counter \ -I debug_counter \ - -I pbh + -I pbh \ + -I nhg CFLAGS_SAI = -I /usr/include/sai @@ -40,9 +41,10 @@ orchagent_SOURCES = \ orchdaemon.cpp \ orch.cpp \ notifications.cpp \ - nhgbase.cpp \ - nhghandler.cpp \ - cbfnhghandler.cpp \ + nhgorch.cpp \ + nhg/nhgbase.cpp \ + nhg/nhghandler.cpp \ + nhg/cbfnhghandler.cpp \ nhgmaporch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ diff --git a/orchagent/cbfnhghandler.cpp b/orchagent/nhg/cbfnhghandler.cpp similarity index 96% rename from orchagent/cbfnhghandler.cpp rename to orchagent/nhg/cbfnhghandler.cpp index 621895a36d..72278d051f 100644 --- a/orchagent/cbfnhghandler.cpp +++ b/orchagent/nhg/cbfnhghandler.cpp @@ -43,7 +43,7 @@ void CbfNhgHandler::doTask(Consumer& consumer) string op = kfvOp(t); bool success; - const auto &cbf_nhg_it = m_syncedNextHopGroups.find(index); + const auto &cbf_nhg_it = m_syncdNextHopGroups.find(index); if (op == SET_COMMAND) { @@ -81,7 +81,7 @@ void CbfNhgHandler::doTask(Consumer& consumer) /* * If the CBF group does not exist, create it. */ - if (cbf_nhg_it == m_syncedNextHopGroups.end()) + if (cbf_nhg_it == m_syncdNextHopGroups.end()) { /* * If we reached the NHG limit, postpone the creation. @@ -93,8 +93,8 @@ void CbfNhgHandler::doTask(Consumer& consumer) } else { - auto cbf_nhg = CbfNhg(index, p.second, selection_map); - success = cbf_nhg.sync(); + auto cbf_nhg = std::make_unique(index, p.second, selection_map); + success = cbf_nhg->sync(); if (success) { @@ -102,12 +102,12 @@ void CbfNhgHandler::doTask(Consumer& consumer) * If the CBF NHG contains temporary NHGs as members, * we have to keep checking for updates. */ - if (cbf_nhg.hasTemps()) + if (cbf_nhg->hasTemps()) { success = false; } - m_syncedNextHopGroups.emplace(index, NhgEntry(move(cbf_nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(move(cbf_nhg))); } } } @@ -116,13 +116,13 @@ void CbfNhgHandler::doTask(Consumer& consumer) */ else { - success = cbf_nhg_it->second.nhg.update(p.second, selection_map); + success = cbf_nhg_it->second.nhg->update(p.second, selection_map); /* * If the CBF NHG has temporary NHGs synced, we need to keep * checking this group in case they are promoted. */ - if (cbf_nhg_it->second.nhg.hasTemps()) + if (cbf_nhg_it->second.nhg->hasTemps()) { success = false; } @@ -143,7 +143,7 @@ void CbfNhgHandler::doTask(Consumer& consumer) success = true; } /* If the group doesn't exist, do nothing. */ - else if (cbf_nhg_it == m_syncedNextHopGroups.end()) + else if (cbf_nhg_it == m_syncdNextHopGroups.end()) { SWSS_LOG_WARN("Deleting inexistent CBF NHG %s", index.c_str()); /* @@ -161,11 +161,11 @@ void CbfNhgHandler::doTask(Consumer& consumer) /* Otherwise, delete it. */ else { - success = cbf_nhg_it->second.nhg.remove(); + success = cbf_nhg_it->second.nhg->remove(); if (success) { - m_syncedNextHopGroups.erase(cbf_nhg_it); + m_syncdNextHopGroups.erase(cbf_nhg_it); } } } diff --git a/orchagent/cbfnhghandler.h b/orchagent/nhg/cbfnhghandler.h similarity index 100% rename from orchagent/cbfnhghandler.h rename to orchagent/nhg/cbfnhghandler.h diff --git a/orchagent/nhgbase.cpp b/orchagent/nhg/nhgbase.cpp similarity index 70% rename from orchagent/nhgbase.cpp rename to orchagent/nhg/nhgbase.cpp index 390d38ccfb..c02dc47195 100644 --- a/orchagent/nhgbase.cpp +++ b/orchagent/nhg/nhgbase.cpp @@ -4,7 +4,7 @@ extern sai_object_id_t gSwitchId; -unsigned NhgBase::m_syncedCount = 0; +unsigned NhgBase::m_syncdCount = 0; /* * Purpose: Destructor. @@ -17,11 +17,10 @@ NhgBase::~NhgBase() { SWSS_LOG_ENTER(); - if (isSynced()) - { - SWSS_LOG_ERROR("Destroying next hop group with SAI ID %lu which is still synced.", m_id); - assert(false); - } + /* + * The group member should be removed from its group before destroying it. + */ + assert(!isSynced()); } /* @@ -35,11 +34,11 @@ void NhgBase::decSyncedCount() { SWSS_LOG_ENTER(); - if (m_syncedCount == 0) + if (m_syncdCount == 0) { SWSS_LOG_ERROR("Decreasing next hop groups count while already 0"); throw logic_error("Decreasing next hop groups count while already 0"); } - --m_syncedCount; + --m_syncdCount; } diff --git a/orchagent/nhgbase.h b/orchagent/nhg/nhgbase.h similarity index 84% rename from orchagent/nhgbase.h rename to orchagent/nhg/nhgbase.h index f718b76f1a..b8647e2d2d 100644 --- a/orchagent/nhgbase.h +++ b/orchagent/nhg/nhgbase.h @@ -49,7 +49,7 @@ class NhgBase */ inline sai_object_id_t getId() const { SWSS_LOG_ENTER(); return m_id; } static inline unsigned getSyncedCount() - { SWSS_LOG_ENTER(); return m_syncedCount; } + { SWSS_LOG_ENTER(); return m_syncdCount; } /* * Check if the next hop group is synced or not. @@ -67,7 +67,7 @@ class NhgBase virtual NextHopGroupKey getNhgKey() const = 0; /* Increment the number of existing groups. */ - static inline void incSyncedCount() { SWSS_LOG_ENTER(); ++m_syncedCount; } + static inline void incSyncedCount() { SWSS_LOG_ENTER(); ++m_syncdCount; } /* Decrement the number of existing groups. */ static void decSyncedCount(); @@ -83,7 +83,7 @@ class NhgBase * decremented when an object is removed. This will also account for the * groups created by RouteOrch. */ - static unsigned m_syncedCount; + static unsigned m_syncdCount; }; /* @@ -116,17 +116,6 @@ class NhgMember NhgMember(const NhgMember&) = delete; void operator=(const NhgMember&) = delete; - virtual ~NhgMember() - { - SWSS_LOG_ENTER(); - - if (isSynced()) - { - SWSS_LOG_ERROR("Deleting next hop group member which is still synced"); - assert(false); - } - } - /* * Sync the NHG member, setting its SAI ID. */ @@ -134,16 +123,8 @@ class NhgMember { SWSS_LOG_ENTER(); - /* - * The SAI ID should be updated from invalid to something valid. - */ - if ((m_id != SAI_NULL_OBJECT_ID) || (gm_id == SAI_NULL_OBJECT_ID)) - { - SWSS_LOG_ERROR("Setting invalid SAI ID %lu to next hop group " - "membeer %s, with current SAI ID %lu", - gm_id, to_string().c_str(), m_id); - throw logic_error("Invalid SAI ID assigned to next hop group member"); - } + /* The SAI ID should be updated from invalid to something valid. */ + assert((m_gm_id == SAI_NULL_OBJECT_ID) && (gm_id != SAI_NULL_OBJECT_ID)); m_id = gm_id; gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); @@ -388,22 +369,24 @@ class NhgCommon : public NhgBase }; /* - * Structure describing a next hop group which NhgHandler owns. Beside having - * a next hop group, we also want to keep a ref count so we don't delete - * objects that are still referenced. + * Structure describing a next hop group which NhgOrch owns. Beside having a + * unique pointer to that next hop group, we also want to keep a ref count so + * NhgOrch knows how many other objects reference the next hop group in order + * not to remove them while still being referenced. */ template struct NhgEntry { - /* The next hop group object in this entry. */ - NhgClass nhg; + /* Pointer to the next hop group. NhgOrch is the sole owner of it. */ + std::unique_ptr nhg; /* Number of external objects referencing this next hop group. */ unsigned ref_count; NhgEntry() = default; - explicit NhgEntry(NhgClass&& _nhg, unsigned int _ref_count = 0) : - nhg(move(_nhg)), ref_count(_ref_count) { SWSS_LOG_ENTER(); } + explicit NhgEntry(std::unique_ptr&& _nhg, + unsigned int _ref_count = 0) : + nhg(std::move(_nhg)), ref_count(_ref_count) {} }; /* @@ -419,7 +402,7 @@ class NhgHandlerCommon inline bool hasNhg(const string &index) const { SWSS_LOG_ENTER(); - return m_syncedNextHopGroups.find(index) != m_syncedNextHopGroups.end(); + return m_syncdNextHopGroups.find(index) != m_syncdNextHopGroups.end(); } /* @@ -427,14 +410,14 @@ class NhgHandlerCommon * exist in the map, a out_of_range eexception will be thrown. */ inline const NhgClass& getNhg(const string &index) const - { SWSS_LOG_ENTER(); return m_syncedNextHopGroups.at(index).nhg; } + { return *m_syncdNextHopGroups.at(index).nhg; } /* Increase the ref count for a NHG given by it's index. */ void incNhgRefCount(const string& index) { SWSS_LOG_ENTER(); - auto& nhg_entry = m_syncedNextHopGroups.at(index); + auto& nhg_entry = m_syncdNextHopGroups.at(index); ++nhg_entry.ref_count; } @@ -443,17 +426,10 @@ class NhgHandlerCommon { SWSS_LOG_ENTER(); - auto& nhg_entry = m_syncedNextHopGroups.at(index); + auto& nhg_entry = m_syncdNextHopGroups.at(index); /* Sanity check so we don't overflow. */ - if (nhg_entry.ref_count == 0) - { - SWSS_LOG_ERROR("Trying to decrement next hop group %s reference " - "count while none are left.", - nhg_entry.nhg.to_string().c_str()); - throw logic_error("Decreasing ref count which is already 0"); - } - + assert(nhg_entry.ref_count > 0); --nhg_entry.ref_count; } @@ -461,5 +437,5 @@ class NhgHandlerCommon /* * Map of synced next hop groups. */ - unordered_map> m_syncedNextHopGroups; + unordered_map> m_syncdNextHopGroups; }; diff --git a/orchagent/nhghandler.cpp b/orchagent/nhg/nhghandler.cpp similarity index 93% rename from orchagent/nhghandler.cpp rename to orchagent/nhg/nhghandler.cpp index 510ff5a763..ae2930d5bb 100644 --- a/orchagent/nhghandler.cpp +++ b/orchagent/nhg/nhghandler.cpp @@ -1,8 +1,8 @@ #include "nhghandler.h" #include "nhgorch.h" #include "neighorch.h" -#include "routeorch.h" #include "crmorch.h" +#include "routeorch.h" #include "bulker.h" #include "logger.h" #include "swssnet.h" @@ -41,7 +41,7 @@ void NhgHandler::doTask(Consumer& consumer) string op = kfvOp(t); bool success = false; - const auto& nhg_it = m_syncedNextHopGroups.find(index); + const auto& nhg_it = m_syncdNextHopGroups.find(index); if (op == SET_COMMAND) { @@ -87,7 +87,7 @@ void NhgHandler::doTask(Consumer& consumer) NextHopGroupKey nhg_key = NextHopGroupKey(nhg_str, weights); /* If the group does not exist, create one. */ - if (nhg_it == m_syncedNextHopGroups.end()) + if (nhg_it == m_syncdNextHopGroups.end()) { /* * If we've reached the NHG limit, we're going to create a temporary @@ -102,10 +102,10 @@ void NhgHandler::doTask(Consumer& consumer) try { - auto nhg = createTempNhg(nhg_key); - if (nhg.sync()) + auto nhg = std::make_unique(createTempNhg(nhg_key)); + if (nhg->sync()) { - m_syncedNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } else { @@ -123,19 +123,19 @@ void NhgHandler::doTask(Consumer& consumer) } else { - auto nhg = Nhg(nhg_key, false); - success = nhg.sync(); + auto nhg = std::make_unique(nhg_key, false); + success = nhg->sync(); if (success) { - m_syncedNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } } } /* If the group exists, update it. */ else { - auto& nhg = nhg_it->second.nhg; + const auto& nhg_ptr = nhg_it->second.nhg; /* * If the update would mandate promoting a temporary next hop @@ -143,7 +143,7 @@ void NhgHandler::doTask(Consumer& consumer) * resources yet, we have to skip it until we have enough * resources. */ - if (nhg.isTemp() && + if (nhg_ptr->isTemp() && (gRouteOrch->getNhgCount() + Nhg::getSyncedCount() >= gRouteOrch->getMaxNhgCount())) { /* @@ -153,12 +153,12 @@ void NhgHandler::doTask(Consumer& consumer) * the new key. Otherwise, this will be a no-op as we have * to wait for resources in order to promote the group. */ - if (!nhg_key.contains(nhg.getKey())) + if (!nhg_key.contains(nhg_ptr->getKey())) { try { /* Create the new temporary next hop group. */ - auto new_nhg = createTempNhg(nhg_key); + auto new_nhg = std::make_unique(createTempNhg(nhg_key)); /* * If we successfully sync the new group, update @@ -166,7 +166,7 @@ void NhgHandler::doTask(Consumer& consumer) * don't mess up the reference counter, as other * objects may already reference it. */ - if (new_nhg.sync()) + if (new_nhg->sync()) { nhg_it->second.nhg = std::move(new_nhg); } @@ -189,10 +189,10 @@ void NhgHandler::doTask(Consumer& consumer) * If the group is temporary but can now be promoted, create and sync a new group for * the desired next hops. */ - else if (nhg.isTemp()) + else if (nhg_ptr->isTemp()) { - auto nhg = Nhg(nhg_key, false); - success = nhg.sync(); + auto nhg = std::make_unique(nhg_key, false); + success = nhg->sync(); if (success) { @@ -206,7 +206,7 @@ void NhgHandler::doTask(Consumer& consumer) /* Common update, when all the requirements are met. */ else { - success = nhg.update(nhg_key); + success = nhg_ptr->update(nhg_key); } } } @@ -225,7 +225,7 @@ void NhgHandler::doTask(Consumer& consumer) success = true; } /* If the group does not exist, do nothing. */ - else if (nhg_it == m_syncedNextHopGroups.end()) + else if (nhg_it == m_syncdNextHopGroups.end()) { SWSS_LOG_INFO("Unable to find group with key %s to remove", index.c_str()); /* Mark the operation as successful to consume it. */ @@ -239,19 +239,19 @@ void NhgHandler::doTask(Consumer& consumer) /* Else, if the group is no more referenced, remove it. */ else { - auto& nhg = nhg_it->second.nhg; + const auto& nhg = nhg_it->second.nhg; - success = nhg.remove(); + success = nhg->remove(); if (success) { - m_syncedNextHopGroups.erase(nhg_it); + m_syncdNextHopGroups.erase(nhg_it); } } } else { - SWSS_LOG_WARN("Unknown operation type %s", op.c_str()); + SWSS_LOG_ERROR("Unknown operation type %s\n", op.c_str()); /* Mark the operation as successful to consume it. */ success = true; } @@ -285,17 +285,17 @@ bool NhgHandler::validateNextHop(const NextHopKey& nh_key) * Iterate through all groups and validate the next hop in those who * contain it. */ - for (auto& it : m_syncedNextHopGroups) + for (auto& it : m_syncdNextHopGroups) { auto& nhg = it.second.nhg; - if (nhg.hasMember(nh_key)) + if (nhg->hasMember(nh_key)) { /* * If sync fails, exit right away, as we expect it to be due to a * raeson for which any other future validations will fail too. */ - if (!nhg.validateNextHop(nh_key)) + if (!nhg->validateNextHop(nh_key)) { SWSS_LOG_ERROR("Failed to validate next hop %s in group %s", nh_key.to_string().c_str(), @@ -325,14 +325,14 @@ bool NhgHandler::invalidateNextHop(const NextHopKey& nh_key) * Iterate through all groups and invalidate the next hop from those who * contain it. */ - for (auto& it : m_syncedNextHopGroups) + for (auto& it : m_syncdNextHopGroups) { auto& nhg = it.second.nhg; - if (nhg.hasMember(nh_key)) + if (nhg->hasMember(nh_key)) { /* If the remove fails, exit right away. */ - if (!nhg.invalidateNextHop(nh_key)) + if (!nhg->invalidateNextHop(nh_key)) { SWSS_LOG_WARN("Failed to invalidate next hop %s from group %s", nh_key.to_string().c_str(), @@ -538,6 +538,7 @@ bool Nhg::sync() else { m_id = nhid; + gNeighOrch->increaseNextHopRefCount(nhgm.getKey()); } } else @@ -576,7 +577,7 @@ bool Nhg::sync() gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); /* Increment the number of synced NHGs. */ - ++m_syncedCount; + ++m_syncdCount; /* * Try creating the next hop group's members over SAI. @@ -655,7 +656,13 @@ bool Nhg::remove() // If the group is temporary, there is nothing to be done - just reset the ID. if (m_is_temp) { - m_id = SAI_NULL_OBJECT_ID; + if (m_id != SAI_NULL_OBJECT_ID) + { + m_id = SAI_NULL_OBJECT_ID; + + const WeightedNhgMember& nhgm = m_members.begin()->second; + gNeighOrch->decreaseNextHopRefCount(nhgm.getKey()); + } return true; } diff --git a/orchagent/nhghandler.h b/orchagent/nhg/nhghandler.h similarity index 100% rename from orchagent/nhghandler.h rename to orchagent/nhg/nhghandler.h diff --git a/orchagent/nhgmaporch.cpp b/orchagent/nhgmaporch.cpp index 45604dd8a7..61d5ab33d1 100644 --- a/orchagent/nhgmaporch.cpp +++ b/orchagent/nhgmaporch.cpp @@ -28,8 +28,6 @@ NhgMapOrch::NhgMapOrch(swss::DBConnector *db, const string &table_name) : Orch(d SWSS_LOG_WARN("Switch does not support NHG maps"); m_max_nhg_map_count = 0; } - - SWSS_LOG_INFO("Maximum number of NHG maps: %lu", m_max_nhg_map_count); } void NhgMapOrch::doTask(Consumer &consumer) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp new file mode 100644 index 0000000000..fbea34a161 --- /dev/null +++ b/orchagent/nhgorch.cpp @@ -0,0 +1,79 @@ +#include "nhgorch.h" + +extern PortsOrch *gPortsOrch; + +NhgOrch::NhgOrch(DBConnector *db, const vector &table_names) : + Orch(db, table_names) +{ + SWSS_LOG_ENTER(); +} + +/* + * Purpose: Perform the operations requested by APPL_DB users. + * Description: Iterate over the untreated operations list and resolve them. + * The operations supported are SET and DEL. If an operation + * could not be resolved, it will either remain in the list, or be + * removed, depending on the case. + * Params: IN consumer - The cosumer object. + * Returns: Nothing. + */ +void NhgOrch::doTask(Consumer& consumer) +{ + SWSS_LOG_ENTER(); + + if (!gPortsOrch->allPortsReady()) + { + return; + } + + string table_name = consumer.getTableName(); + + if (table_name == APP_NEXTHOP_GROUP_TABLE_NAME) + { + nhgHandler.doTask(consumer); + } + else if (table_name == APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) + { + cbfNhgHandler.doTask(consumer); + } +} + +/* + * Purpose: Increase the ref count for a next hop group. + * Description: Increment the ref count for a next hop group by 1. + * Params: IN index - The index of the next hop group. + * Returns: Nothing. + */ +void NhgOrch::incNhgRefCount(const std::string& index) +{ + SWSS_LOG_ENTER(); + + if (nhgHandler.hasNhg(index)) + { + nhgHandler.incNhgRefCount(index); + } + else + { + cbfNhgHandler.incNhgRefCount(index); + } +} + +/* + * Purpose: Decrease the ref count for a next hop group. + * Description: Decrement the ref count for a next hop group by 1. + * Params: IN index - The index of the next hop group. + * Returns: Nothing. + */ +void NhgOrch::decNhgRefCount(const std::string& index) +{ + SWSS_LOG_ENTER(); + + if (nhgHandler.hasNhg(index)) + { + nhgHandler.decNhgRefCount(index); + } + else + { + cbfNhgHandler.decNhgRefCount(index); + } +} \ No newline at end of file diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index 20369c9a0e..c75e0b2c56 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -1,72 +1,37 @@ #pragma once -#include "cbfnhghandler.h" -#include "nhghandler.h" -#include "switchorch.h" +#include "nhg/cbfnhghandler.h" +#include "nhg/nhghandler.h" #include "vector" -#include "portsorch.h" +#include "routeorch.h" using namespace std; -extern PortsOrch *gPortsOrch; +extern RouteOrch *gRouteOrch; +/* + * Next Hop Group Orchestrator class that handles NEXTHOP_GROUP_TABLE + * and CLASS_BASED_NEXT_HOP_GROUP_TABLE updates. + */ class NhgOrch : public Orch { public: - NhgOrch(DBConnector *db, const vector &table_names) : - Orch(db, table_names) - { - SWSS_LOG_ENTER(); - } - - /* - * Get the maximum number of ECMP groups allowed by the switch. - */ - static inline unsigned getMaxNhgCount() - { SWSS_LOG_ENTER(); return m_maxNhgCount; } - /* - * Get the number of next hop groups that are synced. + * Constructor. */ - static inline unsigned getSyncedNhgCount() - { SWSS_LOG_ENTER(); return NhgBase::getSyncedCount(); } - - /* Increase the number of synced next hop groups. */ - static void incSyncedNhgCount() - { - SWSS_LOG_ENTER(); - - if (getSyncedNhgCount() >= m_maxNhgCount) - { - SWSS_LOG_ERROR("Incresing synced next hop group count beyond " - "switch's capabilities"); - throw logic_error("Next hop groups exceed switch's " - "capabilities"); - } - - NhgBase::incSyncedCount(); - } - - /* Decrease the number of next hop groups. */ - static inline void decSyncedNhgCount() - { SWSS_LOG_ENTER(); NhgBase::decSyncedCount(); } + NhgOrch(DBConnector *db, const vector &table_names); - /* - * Check if the next hop group with the given index exists. - */ - inline bool hasNhg(const string &index) const + /* Check if the next hop group given by it's index exists. */ + inline bool hasNhg(const std::string& index) const { - SWSS_LOG_ENTER(); return nhgHandler.hasNhg(index) || cbfNhgHandler.hasNhg(index); } /* * Get the next hop group with the given index. */ - const NhgBase& getNhg(const string &index) const + inline const NhgBase& getNhg(const std::string &index) const { - SWSS_LOG_ENTER(); - try { return nhgHandler.getNhg(index); @@ -77,62 +42,20 @@ class NhgOrch : public Orch } } - /* - * Increase the reference counter for the next hop group with the given - * index. - */ - void incNhgRefCount(const string &index) - { - SWSS_LOG_ENTER(); - - if (nhgHandler.hasNhg(index)) - { - nhgHandler.incNhgRefCount(index); - } - else - { - cbfNhgHandler.incNhgRefCount(index); - } - } + /* Getters / Setters. */ + static inline unsigned getSyncedNhgCount() { return NhgBase::getSyncedCount(); } - /* - * Decrease the reference counter for the next hop group with the given - * index. - */ - void decNhgRefCount(const string &index) + /* Increase / Decrease the number of synced next hop groups. */ + inline void incSyncedNhgCount() { - SWSS_LOG_ENTER(); - - if (nhgHandler.hasNhg(index)) - { - nhgHandler.decNhgRefCount(index); - } - else - { - cbfNhgHandler.decNhgRefCount(index); - } + assert(gRouteOrch->getNhgCount() + NhgBase::getSyncedCount() < gRouteOrch->getMaxNhgCount()); + NhgBase::incSyncedCount(); } + inline void decSyncedNhgCount() { NhgBase::decSyncedCount(); } - void doTask(Consumer &consumer) override - { - SWSS_LOG_ENTER(); - - if (!gPortsOrch->allPortsReady()) - { - return; - } - - string table_name = consumer.getTableName(); - - if (table_name == APP_NEXTHOP_GROUP_TABLE_NAME) - { - nhgHandler.doTask(consumer); - } - else if (table_name == APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) - { - cbfNhgHandler.doTask(consumer); - } - } + /* Increase / Decrease ref count for a NHG given by it's index. */ + void incNhgRefCount(const std::string& index); + void decNhgRefCount(const std::string& index); /* Handling SAI status*/ task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, void *context = nullptr) @@ -149,8 +72,5 @@ class NhgOrch : public Orch CbfNhgHandler cbfNhgHandler; private: - /* - * Switch's maximum number of next hop groups capacity. - */ - static unsigned m_maxNhgCount; + void doTask(Consumer& consumer); }; diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index 3640756a8d..6b72a6c7c9 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -21,8 +21,6 @@ extern sai_switch_api_t* sai_switch_api; extern sai_object_id_t gSwitchId; extern bool gSaiRedisLogRotate; -unsigned NhgOrch::m_maxNhgCount = 0; - extern void syncd_apply_view(); /* * Global orch daemon variables @@ -176,8 +174,8 @@ bool OrchDaemon::init() { APP_ROUTE_TABLE_NAME, routeorch_pri }, { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; - gNhgOrch = new NhgOrch(m_applDb, {APP_NEXTHOP_GROUP_TABLE_NAME, APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME}); gRouteOrch = new RouteOrch(m_applDb, route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch, gSrv6Orch); + gNhgOrch = new NhgOrch(m_applDb, {APP_NEXTHOP_GROUP_TABLE_NAME, APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME}); CoppOrch *copp_orch = new CoppOrch(m_applDb, APP_COPP_TABLE_NAME); TunnelDecapOrch *tunnel_decap_orch = new TunnelDecapOrch(m_applDb, APP_TUNNEL_DECAP_TABLE_NAME); diff --git a/tests/mock_tests/Makefile.am b/tests/mock_tests/Makefile.am index 934a172d19..0b07c9a7d3 100644 --- a/tests/mock_tests/Makefile.am +++ b/tests/mock_tests/Makefile.am @@ -39,9 +39,10 @@ tests_SOURCES = aclorch_ut.cpp \ $(top_srcdir)/orchagent/routeorch.cpp \ $(top_srcdir)/orchagent/mplsrouteorch.cpp \ $(top_srcdir)/orchagent/fgnhgorch.cpp \ - $(top_srcdir)/orchagent/nhgbase.cpp \ - $(top_srcdir)/orchagent/nhghandler.cpp \ - $(top_srcdir)/orchagent/cbfnhghandler.cpp \ + $(top_srcdir)/orchagent/nhgorch.cpp \ + $(top_srcdir)/orchagent/nhg/nhgbase.cpp \ + $(top_srcdir)/orchagent/nhg/nhghandler.cpp \ + $(top_srcdir)/orchagent/nhg/cbfnhghandler.cpp \ $(top_srcdir)/orchagent/nhgmaporch.cpp \ $(top_srcdir)/orchagent/neighorch.cpp \ $(top_srcdir)/orchagent/intfsorch.cpp \ From 360159947d0f4edbbd983ce9c2ca35974aa7ff73 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 19 Oct 2021 13:16:04 +0100 Subject: [PATCH 03/29] Restructure --- orchagent/Makefile.am | 1 - orchagent/nhgorch.cpp | 79 ------------------------------------ orchagent/nhgorch.h | 58 +++++++++++++++++++++++--- tests/mock_tests/Makefile.am | 1 - 4 files changed, 53 insertions(+), 86 deletions(-) delete mode 100644 orchagent/nhgorch.cpp diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index ed30d16434..518c65708a 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -41,7 +41,6 @@ orchagent_SOURCES = \ orchdaemon.cpp \ orch.cpp \ notifications.cpp \ - nhgorch.cpp \ nhg/nhgbase.cpp \ nhg/nhghandler.cpp \ nhg/cbfnhghandler.cpp \ diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp deleted file mode 100644 index fbea34a161..0000000000 --- a/orchagent/nhgorch.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "nhgorch.h" - -extern PortsOrch *gPortsOrch; - -NhgOrch::NhgOrch(DBConnector *db, const vector &table_names) : - Orch(db, table_names) -{ - SWSS_LOG_ENTER(); -} - -/* - * Purpose: Perform the operations requested by APPL_DB users. - * Description: Iterate over the untreated operations list and resolve them. - * The operations supported are SET and DEL. If an operation - * could not be resolved, it will either remain in the list, or be - * removed, depending on the case. - * Params: IN consumer - The cosumer object. - * Returns: Nothing. - */ -void NhgOrch::doTask(Consumer& consumer) -{ - SWSS_LOG_ENTER(); - - if (!gPortsOrch->allPortsReady()) - { - return; - } - - string table_name = consumer.getTableName(); - - if (table_name == APP_NEXTHOP_GROUP_TABLE_NAME) - { - nhgHandler.doTask(consumer); - } - else if (table_name == APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) - { - cbfNhgHandler.doTask(consumer); - } -} - -/* - * Purpose: Increase the ref count for a next hop group. - * Description: Increment the ref count for a next hop group by 1. - * Params: IN index - The index of the next hop group. - * Returns: Nothing. - */ -void NhgOrch::incNhgRefCount(const std::string& index) -{ - SWSS_LOG_ENTER(); - - if (nhgHandler.hasNhg(index)) - { - nhgHandler.incNhgRefCount(index); - } - else - { - cbfNhgHandler.incNhgRefCount(index); - } -} - -/* - * Purpose: Decrease the ref count for a next hop group. - * Description: Decrement the ref count for a next hop group by 1. - * Params: IN index - The index of the next hop group. - * Returns: Nothing. - */ -void NhgOrch::decNhgRefCount(const std::string& index) -{ - SWSS_LOG_ENTER(); - - if (nhgHandler.hasNhg(index)) - { - nhgHandler.decNhgRefCount(index); - } - else - { - cbfNhgHandler.decNhgRefCount(index); - } -} \ No newline at end of file diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index c75e0b2c56..a88d9cf2d5 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -3,10 +3,12 @@ #include "nhg/cbfnhghandler.h" #include "nhg/nhghandler.h" #include "vector" +#include "portsorch.h" #include "routeorch.h" using namespace std; +extern PortsOrch *gPortsOrch; extern RouteOrch *gRouteOrch; /* @@ -19,8 +21,11 @@ class NhgOrch : public Orch /* * Constructor. */ - NhgOrch(DBConnector *db, const vector &table_names); - + NhgOrch(DBConnector *db, const vector &table_names) : + Orch(db, table_names) + { + SWSS_LOG_ENTER(); + } /* Check if the next hop group given by it's index exists. */ inline bool hasNhg(const std::string& index) const { @@ -54,8 +59,32 @@ class NhgOrch : public Orch inline void decSyncedNhgCount() { NhgBase::decSyncedCount(); } /* Increase / Decrease ref count for a NHG given by it's index. */ - void incNhgRefCount(const std::string& index); - void decNhgRefCount(const std::string& index); + void incNhgRefCount(const std::string& index) + { + SWSS_LOG_ENTER(); + + if (nhgHandler.hasNhg(index)) + { + nhgHandler.incNhgRefCount(index); + } + else + { + cbfNhgHandler.incNhgRefCount(index); + } + } + void decNhgRefCount(const std::string& index) + { + SWSS_LOG_ENTER(); + + if (nhgHandler.hasNhg(index)) + { + nhgHandler.decNhgRefCount(index); + } + else + { + cbfNhgHandler.decNhgRefCount(index); + } + } /* Handling SAI status*/ task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, void *context = nullptr) @@ -72,5 +101,24 @@ class NhgOrch : public Orch CbfNhgHandler cbfNhgHandler; private: - void doTask(Consumer& consumer); + void doTask(Consumer& consumer) + { + SWSS_LOG_ENTER(); + + if (!gPortsOrch->allPortsReady()) + { + return; + } + + string table_name = consumer.getTableName(); + + if (table_name == APP_NEXTHOP_GROUP_TABLE_NAME) + { + nhgHandler.doTask(consumer); + } + else if (table_name == APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) + { + cbfNhgHandler.doTask(consumer); + } + } }; diff --git a/tests/mock_tests/Makefile.am b/tests/mock_tests/Makefile.am index 0b07c9a7d3..8541fa5287 100644 --- a/tests/mock_tests/Makefile.am +++ b/tests/mock_tests/Makefile.am @@ -39,7 +39,6 @@ tests_SOURCES = aclorch_ut.cpp \ $(top_srcdir)/orchagent/routeorch.cpp \ $(top_srcdir)/orchagent/mplsrouteorch.cpp \ $(top_srcdir)/orchagent/fgnhgorch.cpp \ - $(top_srcdir)/orchagent/nhgorch.cpp \ $(top_srcdir)/orchagent/nhg/nhgbase.cpp \ $(top_srcdir)/orchagent/nhg/nhghandler.cpp \ $(top_srcdir)/orchagent/nhg/cbfnhghandler.cpp \ From c3c7e781033eeffd34929d8b261193908e4dd1a7 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Fri, 22 Oct 2021 13:32:25 +0100 Subject: [PATCH 04/29] Fix UTs --- tests/test_qos_map.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index b51d5cbb49..4faddc9fbf 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -170,10 +170,10 @@ def test_dscp_to_fc(self, dvs): ] for fvs in maps: - dscp_ps.set('AZURE', swsscommon.FieldValuePairs([fvs])) + self.dscp_ps.set('AZURE', swsscommon.FieldValuePairs([fvs])) time.sleep(1) - assert(asic_qos_map_count == len(asic_db.get_keys('ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP'))) - dscp_ps._del("AZURE") + assert(self.asic_qos_map_count == len(self.asic_db.get_keys('ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP'))) + self.dscp_ps._del("AZURE") # Delete a map that does not exist. Nothing should happen self.dscp_ps._del("AZURE") @@ -211,10 +211,10 @@ def test_exp_to_fc(self, dvs): ] for fvs in maps: - exp_ps.set('AZURE', swsscommon.FieldValuePairs([fvs])) + self.exp_ps.set('AZURE', swsscommon.FieldValuePairs([fvs])) time.sleep(1) - assert(asic_qos_map_count == len(asic_db.get_keys('ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP'))) - exp_ps._del("AZURE") + assert(self.asic_qos_map_count == len(self.asic_db.get_keys('ASIC_STATE:SAI_OBJECT_TYPE_QOS_MAP'))) + self.exp_ps._del("AZURE") # Update the map with valid values exp_map = [(str(i), str(i + 10)) for i in range(0, 8)] From bc54d5746624850750d87690d301db30c5d91253 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Mon, 25 Oct 2021 09:34:52 +0100 Subject: [PATCH 05/29] Restructure code --- orchagent/Makefile.am | 10 +- .../cbfnhghandler.cpp => cbf/cbfnhgorch.cpp} | 29 ++- .../{nhg/cbfnhghandler.h => cbf/cbfnhgorch.h} | 10 +- orchagent/{ => cbf}/nhgmaporch.cpp | 0 orchagent/{ => cbf}/nhgmaporch.h | 0 orchagent/neighorch.cpp | 4 +- orchagent/nhg/nhghandler.h | 116 ------------ orchagent/{nhg => }/nhgbase.cpp | 0 orchagent/{nhg => }/nhgbase.h | 22 +-- orchagent/{nhg/nhghandler.cpp => nhgorch.cpp} | 120 ++++++++---- orchagent/nhgorch.h | 174 +++++++++++------- orchagent/orchdaemon.cpp | 6 +- orchagent/orchdaemon.h | 3 +- orchagent/qosorch.cpp | 2 +- tests/mock_tests/Makefile.am | 8 +- 15 files changed, 252 insertions(+), 252 deletions(-) rename orchagent/{nhg/cbfnhghandler.cpp => cbf/cbfnhgorch.cpp} (97%) rename orchagent/{nhg/cbfnhghandler.h => cbf/cbfnhgorch.h} (95%) rename orchagent/{ => cbf}/nhgmaporch.cpp (100%) rename orchagent/{ => cbf}/nhgmaporch.h (100%) delete mode 100644 orchagent/nhg/nhghandler.h rename orchagent/{nhg => }/nhgbase.cpp (100%) rename orchagent/{nhg => }/nhgbase.h (94%) rename orchagent/{nhg/nhghandler.cpp => nhgorch.cpp} (89%) diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index 518c65708a..9f03aa6a77 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -4,7 +4,7 @@ INCLUDES = -I $(top_srcdir)/lib \ -I flex_counter \ -I debug_counter \ -I pbh \ - -I nhg + -I cbf CFLAGS_SAI = -I /usr/include/sai @@ -41,10 +41,10 @@ orchagent_SOURCES = \ orchdaemon.cpp \ orch.cpp \ notifications.cpp \ - nhg/nhgbase.cpp \ - nhg/nhghandler.cpp \ - nhg/cbfnhghandler.cpp \ - nhgmaporch.cpp \ + nhgbase.cpp \ + nhgorch.cpp \ + cbf/cbfnhgorch.cpp \ + cbf/nhgmaporch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ neighorch.cpp \ diff --git a/orchagent/nhg/cbfnhghandler.cpp b/orchagent/cbf/cbfnhgorch.cpp similarity index 97% rename from orchagent/nhg/cbfnhghandler.cpp rename to orchagent/cbf/cbfnhgorch.cpp index 72278d051f..8b83ff7867 100644 --- a/orchagent/nhg/cbfnhghandler.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -1,4 +1,4 @@ -#include "cbfnhghandler.h" +#include "cbfnhgorch.h" #include "crmorch.h" #include "bulker.h" #include "tokenize.h" @@ -17,6 +17,12 @@ extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern size_t gMaxBulkSize; +CbfNhgOrch::CbfNhgOrch(DBConnector *db, string tableName) : + Orch(db, tableName) +{ + SWSS_LOG_ENTER(); +} + /* * Purpose: Perform the operations requested by APPL_DB users. * @@ -29,10 +35,15 @@ extern size_t gMaxBulkSize; * * Returns: Nothing. */ -void CbfNhgHandler::doTask(Consumer& consumer) +void CbfNhgOrch::doTask(Consumer& consumer) { SWSS_LOG_ENTER(); + if (!gPortsOrch->allPortsReady()) + { + return; + } + auto it = consumer.m_toSync.begin(); while (it != consumer.m_toSync.end()) @@ -198,7 +209,7 @@ void CbfNhgHandler::doTask(Consumer& consumer) * valid or not * - the second element is a vector of members */ -pair> CbfNhgHandler::getMembers(const string &members) +pair> CbfNhgOrch::getMembers(const string &members) { SWSS_LOG_ENTER(); @@ -536,7 +547,7 @@ bool CbfNhg::update(const vector &members, const string &selection_map) return false; } - auto status = sai_next_hop_group_api->set_next_hop_group_attribute( m_id, &nhg_attr); + auto status = sai_next_hop_group_api->set_next_hop_group_attribute(m_id, &nhg_attr); if (status != SAI_STATUS_SUCCESS) { @@ -599,14 +610,14 @@ bool CbfNhg::syncMembers(const set &members) /* * Check if the group exists in NhgOrch. */ - if (!gNhgOrch->nhgHandler.hasNhg(key)) + if (!gNhgOrch->hasNhg(key)) { SWSS_LOG_ERROR("Next hop group %s in CBF next hop group %s does " "not exist", key.c_str(), m_key.c_str()); return false; } - const auto &nhg = gNhgOrch->nhgHandler.getNhg(key); + const auto &nhg = gNhgOrch->getNhg(key); /* * Check if the group is synced. @@ -714,7 +725,7 @@ void CbfNhgMember::sync(sai_object_id_t gm_id) SWSS_LOG_ENTER(); NhgMember::sync(gm_id); - gNhgOrch->nhgHandler.incNhgRefCount(m_key); + gNhgOrch->incNhgRefCount(m_key); } /* @@ -746,7 +757,7 @@ bool CbfNhgMember::updateNhAttr() /* * Set the attribute over SAI. */ - auto status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_id, &attr); + auto status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_gm_id, &attr); return status == SAI_STATUS_SUCCESS; } @@ -764,7 +775,7 @@ void CbfNhgMember::remove() SWSS_LOG_ENTER(); NhgMember::remove(); - gNhgOrch->nhgHandler.decNhgRefCount(m_key); + gNhgOrch->decNhgRefCount(m_key); } /* diff --git a/orchagent/nhg/cbfnhghandler.h b/orchagent/cbf/cbfnhgorch.h similarity index 95% rename from orchagent/nhg/cbfnhghandler.h rename to orchagent/cbf/cbfnhgorch.h index af2dbb30af..e4631cb9cb 100644 --- a/orchagent/nhg/cbfnhghandler.h +++ b/orchagent/cbf/cbfnhgorch.h @@ -1,7 +1,6 @@ #pragma once #include "nhgbase.h" -#include "nhghandler.h" using namespace std; @@ -86,11 +85,16 @@ class CbfNhg : public NhgCommon bool hasSameMembers(const vector &members) const; }; -class CbfNhgHandler : public NhgHandlerCommon +class CbfNhgOrch : public Orch, public NhgOrchCommon { public: - void doTask(Consumer &consumer); + /* + * Constructor. + */ + CbfNhgOrch(DBConnector *db, string tableName); private: static pair> getMembers(const string &members); + + void doTask(Consumer &consumer); }; diff --git a/orchagent/nhgmaporch.cpp b/orchagent/cbf/nhgmaporch.cpp similarity index 100% rename from orchagent/nhgmaporch.cpp rename to orchagent/cbf/nhgmaporch.cpp diff --git a/orchagent/nhgmaporch.h b/orchagent/cbf/nhgmaporch.h similarity index 100% rename from orchagent/nhgmaporch.h rename to orchagent/cbf/nhgmaporch.h diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index 44c10d818a..c1eba4c0e3 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -334,7 +334,7 @@ bool NeighOrch::setNextHopFlag(const NextHopKey &nexthop, const uint32_t nh_flag { case NHFLAGS_IFDOWN: rc = gRouteOrch->invalidnexthopinNextHopGroup(nexthop, count); - rc &= gNhgOrch->nhgHandler.invalidateNextHop(nexthop); + rc &= gNhgOrch->invalidateNextHop(nexthop); break; default: assert(0); @@ -364,7 +364,7 @@ bool NeighOrch::clearNextHopFlag(const NextHopKey &nexthop, const uint32_t nh_fl { case NHFLAGS_IFDOWN: rc = gRouteOrch->validnexthopinNextHopGroup(nexthop, count); - rc &= gNhgOrch->nhgHandler.validateNextHop(nexthop); + rc &= gNhgOrch->validateNextHop(nexthop); break; default: assert(0); diff --git a/orchagent/nhg/nhghandler.h b/orchagent/nhg/nhghandler.h deleted file mode 100644 index 301ef398a4..0000000000 --- a/orchagent/nhg/nhghandler.h +++ /dev/null @@ -1,116 +0,0 @@ -#pragma once - -#include "nhgbase.h" - -using namespace std; - -class WeightedNhgMember : public NhgMember -{ -public: - /* Constructors / Assignment operators. */ - WeightedNhgMember(const NextHopKey& nh_key) : - NhgMember(nh_key) {} - - WeightedNhgMember(WeightedNhgMember&& nhgm) : - NhgMember(move(nhgm)) {} - - /* Destructor. */ - ~WeightedNhgMember(); - - /* Update member's weight and update the SAI attribute as well. */ - bool updateWeight(uint32_t weight); - - /* Sync / Remove. */ - void sync(sai_object_id_t gm_id) override; - void remove() override; - - /* Getters / Setters. */ - inline uint32_t getWeight() const { return m_key.weight; } - sai_object_id_t getNhId() const; - - /* Check if the next hop is labeled. */ - inline bool isLabeled() const { return !m_key.label_stack.empty(); } - - /* Convert member's details to string. */ - string to_string() const override - { - return m_key.to_string() + - ", SAI ID: " + std::to_string(m_id); - } -}; - -/* - * Nhg class representing a next hop group object. - */ -class Nhg : public NhgCommon -{ -public: - /* Constructors. */ - explicit Nhg(const NextHopGroupKey& key, bool is_temp); - - Nhg(Nhg&& nhg) : - NhgCommon(move(nhg)), m_is_temp(nhg.m_is_temp) - { SWSS_LOG_ENTER(); } - - Nhg& operator=(Nhg&& nhg); - - ~Nhg() { SWSS_LOG_ENTER(); remove(); } - - /* Sync the group, creating the group's and members SAI IDs. */ - bool sync() override; - - /* Remove the group, reseting the group's and members SAI IDs. */ - bool remove() override; - - /* - * Update the group based on a new next hop group key. This will also - * perform any sync / remove necessary. - */ - bool update(const NextHopGroupKey& nhg_key); - - /* Validate a next hop in the group, syncing it. */ - bool validateNextHop(const NextHopKey& nh_key); - - /* Invalidate a next hop in the group, removing it. */ - bool invalidateNextHop(const NextHopKey& nh_key); - - /* Getters / Setters. */ - inline bool isTemp() const override { return m_is_temp; } - inline void setTemp(bool is_temp) { m_is_temp = is_temp; } - - NextHopGroupKey getNhgKey() const override { return m_key; } - - /* Convert NHG's details to a string. */ - string to_string() const override - { - return m_key.to_string() + ", SAI ID: " + std::to_string(m_id); - } - -private: - /* Whether the group is temporary or not. */ - bool m_is_temp; - - /* Add group's members over the SAI API for the given keys. */ - bool syncMembers(const set& nh_keys) override; - - /* Create the attributes vector for a next hop group member. */ - vector createNhgmAttrs( - const WeightedNhgMember& nhgm) const override; -}; - -/* - * Next Hop Group Orchestrator class that handles NEXT_HOP_GROUP_TABLE - * updates. - */ -class NhgHandler : public NhgHandlerCommon -{ -public: - /* Add a temporary next hop group when resources are exhausted. */ - Nhg createTempNhg(const NextHopGroupKey& nhg_key); - - /* Validate / Invalidate a next hop. */ - bool validateNextHop(const NextHopKey& nh_key); - bool invalidateNextHop(const NextHopKey& nh_key); - - void doTask(Consumer &consumer); -}; diff --git a/orchagent/nhg/nhgbase.cpp b/orchagent/nhgbase.cpp similarity index 100% rename from orchagent/nhg/nhgbase.cpp rename to orchagent/nhgbase.cpp diff --git a/orchagent/nhg/nhgbase.h b/orchagent/nhgbase.h similarity index 94% rename from orchagent/nhg/nhgbase.h rename to orchagent/nhgbase.h index b8647e2d2d..f8cb6e1cfe 100644 --- a/orchagent/nhg/nhgbase.h +++ b/orchagent/nhgbase.h @@ -95,17 +95,17 @@ class NhgMember { public: explicit NhgMember(const Key &key) : - m_key(key), m_id(SAI_NULL_OBJECT_ID) { SWSS_LOG_ENTER(); } + m_key(key), m_gm_id(SAI_NULL_OBJECT_ID) { SWSS_LOG_ENTER(); } - NhgMember(NhgMember &&nhgm) : m_key(move(nhgm.m_key)), m_id(nhgm.m_id) - { SWSS_LOG_ENTER(); nhgm.m_id = SAI_NULL_OBJECT_ID; } + NhgMember(NhgMember &&nhgm) : m_key(move(nhgm.m_key)), m_gm_id(nhgm.m_gm_id) + { SWSS_LOG_ENTER(); nhgm.m_gm_id = SAI_NULL_OBJECT_ID; } NhgMember& operator=(NhgMember &&nhgm) { SWSS_LOG_ENTER(); swap(m_key, nhgm.m_key); - swap(m_id, nhgm.m_id); + swap(m_gm_id, nhgm.m_gm_id); return *this; } @@ -126,7 +126,7 @@ class NhgMember /* The SAI ID should be updated from invalid to something valid. */ assert((m_gm_id == SAI_NULL_OBJECT_ID) && (gm_id != SAI_NULL_OBJECT_ID)); - m_id = gm_id; + m_gm_id = gm_id; gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); } @@ -145,7 +145,7 @@ class NhgMember return; } - m_id = SAI_NULL_OBJECT_ID; + m_gm_id = SAI_NULL_OBJECT_ID; gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); } @@ -153,12 +153,12 @@ class NhgMember * Getters. */ inline Key getKey() const { return m_key; } - inline sai_object_id_t getId() const { return m_id; } + inline sai_object_id_t getId() const { return m_gm_id; } /* * Check whether the group is synced. */ - inline bool isSynced() const { return m_id != SAI_NULL_OBJECT_ID; } + inline bool isSynced() const { return m_gm_id != SAI_NULL_OBJECT_ID; } /* * Get a string form of the member. @@ -174,7 +174,7 @@ class NhgMember /* * The SAI ID of this NHG member. */ - sai_object_id_t m_id; + sai_object_id_t m_gm_id; }; /* @@ -390,10 +390,10 @@ struct NhgEntry }; /* - * Class providing the common functionality shared by all NhgHandler classes. + * Class providing the common functionality shared by all NhgOrch classes. */ template -class NhgHandlerCommon +class NhgOrchCommon { public: /* diff --git a/orchagent/nhg/nhghandler.cpp b/orchagent/nhgorch.cpp similarity index 89% rename from orchagent/nhg/nhghandler.cpp rename to orchagent/nhgorch.cpp index ae2930d5bb..e7cccf48fb 100644 --- a/orchagent/nhg/nhghandler.cpp +++ b/orchagent/nhgorch.cpp @@ -1,4 +1,3 @@ -#include "nhghandler.h" #include "nhgorch.h" #include "neighorch.h" #include "crmorch.h" @@ -18,6 +17,12 @@ extern size_t gMaxBulkSize; extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern sai_next_hop_api_t* sai_next_hop_api; +NhgOrch::NhgOrch(DBConnector *db, string tableName) : + Orch(db, tableName) +{ + SWSS_LOG_ENTER(); +} + /* * Purpose: Perform the operations requested by APPL_DB users. * Description: Iterate over the untreated operations list and resolve them. @@ -27,10 +32,15 @@ extern sai_next_hop_api_t* sai_next_hop_api; * Params: IN consumer - The cosumer object. * Returns: Nothing. */ -void NhgHandler::doTask(Consumer& consumer) +void NhgOrch::doTask(Consumer& consumer) { SWSS_LOG_ENTER(); + if (!gPortsOrch->allPortsReady()) + { + return; + } + auto it = consumer.m_toSync.begin(); while (it != consumer.m_toSync.end()) @@ -96,16 +106,16 @@ void NhgHandler::doTask(Consumer& consumer) * to be kept in the sync list so we keep trying to create the * actual group when there are enough resources. */ - if (gRouteOrch->getNhgCount() + Nhg::getSyncedCount() >= gRouteOrch->getMaxNhgCount()) + if (gRouteOrch->getNhgCount() + NextHopGroup::getSyncedCount() >= gRouteOrch->getMaxNhgCount()) { SWSS_LOG_DEBUG("Next hop group count reached its limit."); try { - auto nhg = std::make_unique(createTempNhg(nhg_key)); + auto nhg = std::make_unique(createTempNhg(nhg_key)); if (nhg->sync()) { - m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } else { @@ -123,12 +133,12 @@ void NhgHandler::doTask(Consumer& consumer) } else { - auto nhg = std::make_unique(nhg_key, false); + auto nhg = std::make_unique(nhg_key, false); success = nhg->sync(); if (success) { - m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); + m_syncdNextHopGroups.emplace(index, NhgEntry(std::move(nhg))); } } } @@ -144,7 +154,7 @@ void NhgHandler::doTask(Consumer& consumer) * resources. */ if (nhg_ptr->isTemp() && - (gRouteOrch->getNhgCount() + Nhg::getSyncedCount() >= gRouteOrch->getMaxNhgCount())) + (gRouteOrch->getNhgCount() + NextHopGroup::getSyncedCount() >= gRouteOrch->getMaxNhgCount())) { /* * If the group was updated in such way that the previously @@ -158,7 +168,7 @@ void NhgHandler::doTask(Consumer& consumer) try { /* Create the new temporary next hop group. */ - auto new_nhg = std::make_unique(createTempNhg(nhg_key)); + auto new_nhg = std::make_unique(createTempNhg(nhg_key)); /* * If we successfully sync the new group, update @@ -191,7 +201,7 @@ void NhgHandler::doTask(Consumer& consumer) */ else if (nhg_ptr->isTemp()) { - auto nhg = std::make_unique(nhg_key, false); + auto nhg = std::make_unique(nhg_key, false); success = nhg->sync(); if (success) @@ -277,7 +287,7 @@ void NhgHandler::doTask(Consumer& consumer) * containing groups; * false, otherwise. */ -bool NhgHandler::validateNextHop(const NextHopKey& nh_key) +bool NhgOrch::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); @@ -317,7 +327,7 @@ bool NhgHandler::validateNextHop(const NextHopKey& nh_key) * containing groups; * false, otherwise. */ -bool NhgHandler::invalidateNextHop(const NextHopKey& nh_key) +bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); @@ -345,6 +355,46 @@ bool NhgHandler::invalidateNextHop(const NextHopKey& nh_key) return true; } +/* + * Purpose: Increase the ref count for a next hop group. + * Description: Increment the ref count for a next hop group by 1. + * Params: IN index - The index of the next hop group. + * Returns: Nothing. + */ +void NhgOrch::incNhgRefCount(const std::string& index) +{ + SWSS_LOG_ENTER(); + + try + { + NhgOrchCommon::incNhgRefCount(index); + } + catch(const std::out_of_range &e) + { + gCbfNhgOrch->incNhgRefCount(index); + } +} + +/* + * Purpose: Decrease the ref count for a next hop group. + * Description: Decrement the ref count for a next hop group by 1. + * Params: IN index - The index of the next hop group. + * Returns: Nothing. + */ +void NhgOrch::decNhgRefCount(const std::string& index) +{ + SWSS_LOG_ENTER(); + + try + { + NhgOrchCommon::decNhgRefCount(index); + } + catch(const std::out_of_range &e) + { + gCbfNhgOrch->decNhgRefCount(index); + } +} + /* * Purpose: Get the next hop ID of the member. * Description: Get the SAI ID of the next hop from NeighOrch. @@ -352,7 +402,7 @@ bool NhgHandler::invalidateNextHop(const NextHopKey& nh_key) * Returns: The SAI ID of the next hop, or SAI_NULL_OBJECT_ID if the next * hop is not valid. */ -sai_object_id_t WeightedNhgMember::getNhId() const +sai_object_id_t NextHopGroupMember::getNhId() const { SWSS_LOG_ENTER(); @@ -390,7 +440,7 @@ sai_object_id_t WeightedNhgMember::getNhId() const * Returns: true, if the operation was successful; * false, otherwise. */ -bool WeightedNhgMember::updateWeight(uint32_t weight) +bool NextHopGroupMember::updateWeight(uint32_t weight) { SWSS_LOG_ENTER(); @@ -404,7 +454,7 @@ bool WeightedNhgMember::updateWeight(uint32_t weight) nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; nhgm_attr.value.s32 = m_key.weight; - sai_status_t status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_id, &nhgm_attr); + sai_status_t status = sai_next_hop_group_api->set_next_hop_group_member_attribute(m_gm_id, &nhgm_attr); success = status == SAI_STATUS_SUCCESS; } @@ -418,7 +468,7 @@ bool WeightedNhgMember::updateWeight(uint32_t weight) * Params: IN gm_id - The group member SAI ID to set. * Returns: Nothing. */ -void WeightedNhgMember::sync(sai_object_id_t gm_id) +void NextHopGroupMember::sync(sai_object_id_t gm_id) { SWSS_LOG_ENTER(); @@ -433,7 +483,7 @@ void WeightedNhgMember::sync(sai_object_id_t gm_id) * Params: None. * Returns: Nothing. */ -void WeightedNhgMember::remove() +void NextHopGroupMember::remove() { SWSS_LOG_ENTER(); @@ -448,7 +498,7 @@ void WeightedNhgMember::remove() * Params: None. * Returns: Nothing. */ -WeightedNhgMember::~WeightedNhgMember() +NextHopGroupMember::~NextHopGroupMember() { SWSS_LOG_ENTER(); @@ -473,14 +523,14 @@ WeightedNhgMember::~WeightedNhgMember() * Params: IN key - The next hop group's key. * Returns: Nothing. */ -Nhg::Nhg(const NextHopGroupKey& key, bool is_temp) : NhgCommon(key), m_is_temp(is_temp) +NextHopGroup::NextHopGroup(const NextHopGroupKey& key, bool is_temp) : NhgCommon(key), m_is_temp(is_temp) { SWSS_LOG_ENTER(); /* Parse the key and create the members. */ - for (const auto &it : m_key.getNextHops()) + for (const auto& it : m_key.getNextHops()) { - m_members.emplace(it, WeightedNhgMember(it)); + m_members.emplace(it, NextHopGroupMember(it)); } } @@ -490,7 +540,7 @@ Nhg::Nhg(const NextHopGroupKey& key, bool is_temp) : NhgCommon(key), m_is_temp(i * Params: IN nhg - The rvalue object to swap with. * Returns: Referene to this object. */ -Nhg& Nhg::operator=(Nhg&& nhg) +NextHopGroup& NextHopGroup::operator=(NextHopGroup&& nhg) { SWSS_LOG_ENTER(); @@ -511,7 +561,7 @@ Nhg& Nhg::operator=(Nhg&& nhg) * Returns: true, if the operation was successful; * false, otherwise. */ -bool Nhg::sync() +bool NextHopGroup::sync() { SWSS_LOG_ENTER(); @@ -527,7 +577,7 @@ bool Nhg::sync() */ if (m_is_temp) { - const WeightedNhgMember& nhgm = m_members.begin()->second; + const NextHopGroupMember& nhgm = m_members.begin()->second; sai_object_id_t nhid = nhgm.getNhId(); if (nhid == SAI_NULL_OBJECT_ID) @@ -603,7 +653,7 @@ bool Nhg::sync() * Params: IN index - The CP index of the next hop group. * Returns: The created temporary next hop group. */ -Nhg NhgHandler::createTempNhg(const NextHopGroupKey& nhg_key) +NextHopGroup NhgOrch::createTempNhg(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); @@ -636,7 +686,7 @@ Nhg NhgHandler::createTempNhg(const NextHopGroupKey& nhg_key) advance(it, rand() % valid_nhs.size()); /* Create the temporary group. */ - Nhg nhg(NextHopGroupKey(it->to_string()), true); + NextHopGroup nhg(NextHopGroupKey(it->to_string()), true); return nhg; } @@ -649,7 +699,7 @@ Nhg NhgHandler::createTempNhg(const NextHopGroupKey& nhg_key) * Returns: true, if the operation was successful; * false, otherwise */ -bool Nhg::remove() +bool NextHopGroup::remove() { SWSS_LOG_ENTER(); @@ -660,7 +710,7 @@ bool Nhg::remove() { m_id = SAI_NULL_OBJECT_ID; - const WeightedNhgMember& nhgm = m_members.begin()->second; + const NextHopGroupMember& nhgm = m_members.begin()->second; gNeighOrch->decreaseNextHopRefCount(nhgm.getKey()); } return true; @@ -679,7 +729,7 @@ bool Nhg::remove() * Returns: true, if the members were added succesfully; * false, otherwise. */ -bool Nhg::syncMembers(const std::set& nh_keys) +bool NextHopGroup::syncMembers(const std::set& nh_keys) { SWSS_LOG_ENTER(); @@ -696,7 +746,7 @@ bool Nhg::syncMembers(const std::set& nh_keys) for (const auto& nh_key : nh_keys) { - WeightedNhgMember& nhgm = m_members.at(nh_key); + NextHopGroupMember& nhgm = m_members.at(nh_key); /* If the member is already synced, continue. */ if (nhgm.isSynced()) @@ -768,7 +818,7 @@ bool Nhg::syncMembers(const std::set& nh_keys) * Returns: true, if the operation was successful; * false, otherwise. */ -bool Nhg::update(const NextHopGroupKey& nhg_key) +bool NextHopGroup::update(const NextHopGroupKey& nhg_key) { SWSS_LOG_ENTER(); @@ -824,7 +874,7 @@ bool Nhg::update(const NextHopGroupKey& nhg_key) /* Add any new members to the group. */ for (const auto& it : new_nh_keys) { - m_members.emplace(it, WeightedNhgMember(it)); + m_members.emplace(it, NextHopGroupMember(it)); } /* @@ -847,7 +897,7 @@ bool Nhg::update(const NextHopGroupKey& nhg_key) * Params: IN nhgm - The next hop group member. * Returns: The attributes vector for the given next hop. */ -vector Nhg::createNhgmAttrs(const WeightedNhgMember& nhgm) const +vector NextHopGroup::createNhgmAttrs(const NextHopGroupMember& nhgm) const { SWSS_LOG_ENTER(); @@ -879,7 +929,7 @@ vector Nhg::createNhgmAttrs(const WeightedNhgMember& nhgm) cons * Returns: true, if the operation was successful; * false, otherwise. */ -bool Nhg::validateNextHop(const NextHopKey& nh_key) +bool NextHopGroup::validateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); @@ -893,7 +943,7 @@ bool Nhg::validateNextHop(const NextHopKey& nh_key) * Returns: true, if the operation was successful; * false, otherwise. */ -bool Nhg::invalidateNextHop(const NextHopKey& nh_key) +bool NextHopGroup::invalidateNextHop(const NextHopKey& nh_key) { SWSS_LOG_ENTER(); diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index a88d9cf2d5..4c4d5fa456 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -1,7 +1,6 @@ #pragma once -#include "nhg/cbfnhghandler.h" -#include "nhg/nhghandler.h" +#include "cbf/cbfnhgorch.h" #include "vector" #include "portsorch.h" #include "routeorch.h" @@ -10,26 +9,117 @@ using namespace std; extern PortsOrch *gPortsOrch; extern RouteOrch *gRouteOrch; +extern CbfNhgOrch *gCbfNhgOrch; + +class NextHopGroupMember : public NhgMember +{ +public: + /* Constructors / Assignment operators. */ + NextHopGroupMember(const NextHopKey& nh_key) : + NhgMember(nh_key) {} + + NextHopGroupMember(NextHopGroupMember&& nhgm) : + NhgMember(move(nhgm)) {} + + /* Destructor. */ + ~NextHopGroupMember(); + + /* Update member's weight and update the SAI attribute as well. */ + bool updateWeight(uint32_t weight); + + /* Sync / Remove. */ + void sync(sai_object_id_t gm_id) override; + void remove() override; + + /* Getters / Setters. */ + inline uint32_t getWeight() const { return m_key.weight; } + sai_object_id_t getNhId() const; + + /* Check if the next hop is labeled. */ + inline bool isLabeled() const { return !m_key.label_stack.empty(); } + + /* Convert member's details to string. */ + string to_string() const override + { + return m_key.to_string() + ", SAI ID: " + std::to_string(m_gm_id); + } +}; /* - * Next Hop Group Orchestrator class that handles NEXTHOP_GROUP_TABLE - * and CLASS_BASED_NEXT_HOP_GROUP_TABLE updates. + * NextHopGroup class representing a next hop group object. */ -class NhgOrch : public Orch +class NextHopGroup : public NhgCommon { public: + /* Constructors. */ + explicit NextHopGroup(const NextHopGroupKey& key, bool is_temp); + + NextHopGroup(NextHopGroup&& nhg) : + NhgCommon(move(nhg)), m_is_temp(nhg.m_is_temp) + { SWSS_LOG_ENTER(); } + + NextHopGroup& operator=(NextHopGroup&& nhg); + + /* Destructor. */ + virtual ~NextHopGroup() { remove(); } + + /* Sync the group, creating the group's and members SAI IDs. */ + bool sync() override; + + /* Remove the group, reseting the group's and members SAI IDs. */ + bool remove() override; + /* - * Constructor. + * Update the group based on a new next hop group key. This will also + * perform any sync / remove necessary. */ - NhgOrch(DBConnector *db, const vector &table_names) : - Orch(db, table_names) + bool update(const NextHopGroupKey& nhg_key); + + /* Validate a next hop in the group, syncing it. */ + bool validateNextHop(const NextHopKey& nh_key); + + /* Invalidate a next hop in the group, removing it. */ + bool invalidateNextHop(const NextHopKey& nh_key); + + /* Getters / Setters. */ + inline bool isTemp() const override { return m_is_temp; } + + NextHopGroupKey getNhgKey() const override { return m_key; } + + /* Convert NHG's details to a string. */ + std::string to_string() const override { - SWSS_LOG_ENTER(); + return m_key.to_string() + ", SAI ID: " + std::to_string(m_id); } + +private: + /* Whether the group is temporary or not. */ + bool m_is_temp; + + /* Add group's members over the SAI API for the given keys. */ + bool syncMembers(const set& nh_keys) override; + + /* Create the attributes vector for a next hop group member. */ + vector createNhgmAttrs( + const NextHopGroupMember& nhgm) const override; +}; + +/* + * Next Hop Group Orchestrator class that handles NEXTHOP_GROUP_TABLE + * updates. + */ +class NhgOrch : public NhgOrchCommon, public Orch +{ +public: + /* + * Constructor. + */ + NhgOrch(DBConnector *db, string tableName); + /* Check if the next hop group given by it's index exists. */ inline bool hasNhg(const std::string& index) const { - return nhgHandler.hasNhg(index) || cbfNhgHandler.hasNhg(index); + return NhgOrchCommon::hasNhg(index) || gCbfNhgOrch->hasNhg(index); } /* @@ -39,14 +129,17 @@ class NhgOrch : public Orch { try { - return nhgHandler.getNhg(index); + return NhgOrchCommon::getNhg(index); } catch(const std::out_of_range &e) { - return cbfNhgHandler.getNhg(index); + return gCbfNhgOrch->getNhg(index); } } + /* Add a temporary next hop group when resources are exhausted. */ + NextHopGroup createTempNhg(const NextHopGroupKey& nhg_key); + /* Getters / Setters. */ static inline unsigned getSyncedNhgCount() { return NhgBase::getSyncedCount(); } @@ -59,32 +152,12 @@ class NhgOrch : public Orch inline void decSyncedNhgCount() { NhgBase::decSyncedCount(); } /* Increase / Decrease ref count for a NHG given by it's index. */ - void incNhgRefCount(const std::string& index) - { - SWSS_LOG_ENTER(); + void incNhgRefCount(const std::string& index); + void decNhgRefCount(const std::string& index); - if (nhgHandler.hasNhg(index)) - { - nhgHandler.incNhgRefCount(index); - } - else - { - cbfNhgHandler.incNhgRefCount(index); - } - } - void decNhgRefCount(const std::string& index) - { - SWSS_LOG_ENTER(); - - if (nhgHandler.hasNhg(index)) - { - nhgHandler.decNhgRefCount(index); - } - else - { - cbfNhgHandler.decNhgRefCount(index); - } - } + /* Validate / Invalidate a next hop. */ + bool validateNextHop(const NextHopKey& nh_key); + bool invalidateNextHop(const NextHopKey& nh_key); /* Handling SAI status*/ task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, void *context = nullptr) @@ -93,32 +166,7 @@ class NhgOrch : public Orch { return Orch::handleSaiRemoveStatus(api, status, context); } bool parseHandleSaiStatusFailure(task_process_status status) { return Orch::parseHandleSaiStatusFailure(status); } - - /* - * Handlers dealing with the (non) CBF operations. - */ - NhgHandler nhgHandler; - CbfNhgHandler cbfNhgHandler; private: - void doTask(Consumer& consumer) - { - SWSS_LOG_ENTER(); - - if (!gPortsOrch->allPortsReady()) - { - return; - } - - string table_name = consumer.getTableName(); - - if (table_name == APP_NEXTHOP_GROUP_TABLE_NAME) - { - nhgHandler.doTask(consumer); - } - else if (table_name == APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME) - { - cbfNhgHandler.doTask(consumer); - } - } + void doTask(Consumer& consumer); }; diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index 6b72a6c7c9..f01a6ded7a 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -33,6 +33,7 @@ NeighOrch *gNeighOrch; RouteOrch *gRouteOrch; NhgOrch *gNhgOrch; NhgMapOrch *gNhgMapOrch; +CbfNhgOrch *gCbfNhgOrch; FgNhgOrch *gFgNhgOrch; AclOrch *gAclOrch; PbhOrch *gPbhOrch; @@ -175,7 +176,8 @@ bool OrchDaemon::init() { APP_LABEL_ROUTE_TABLE_NAME, routeorch_pri } }; gRouteOrch = new RouteOrch(m_applDb, route_tables, gSwitchOrch, gNeighOrch, gIntfsOrch, vrf_orch, gFgNhgOrch, gSrv6Orch); - gNhgOrch = new NhgOrch(m_applDb, {APP_NEXTHOP_GROUP_TABLE_NAME, APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME}); + gNhgOrch = new NhgOrch(m_applDb, APP_NEXTHOP_GROUP_TABLE_NAME); + gCbfNhgOrch = new CbfNhgOrch(m_applDb, APP_CLASS_BASED_NEXT_HOP_GROUP_TABLE_NAME); CoppOrch *copp_orch = new CoppOrch(m_applDb, APP_COPP_TABLE_NAME); TunnelDecapOrch *tunnel_decap_orch = new TunnelDecapOrch(m_applDb, APP_TUNNEL_DECAP_TABLE_NAME); @@ -315,7 +317,7 @@ bool OrchDaemon::init() * when iterating ConsumerMap. This is ensured implicitly by the order of keys in ordered map. * For cases when Orch has to process tables in specific order, like PortsOrch during warm start, it has to override Orch::doTask() */ - m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, mux_orch, mux_cb_orch, gIntfsOrch, gNeighOrch, gNhgMapOrch, gNhgOrch, gRouteOrch, copp_orch, qos_orch, wm_orch, policer_orch, tunnel_decap_orch, sflow_orch, debug_counter_orch, gMacsecOrch, gBfdOrch, gSrv6Orch}; + m_orchList = { gSwitchOrch, gCrmOrch, gPortsOrch, gBufferOrch, mux_orch, mux_cb_orch, gIntfsOrch, gNeighOrch, gNhgMapOrch, gNhgOrch, gCbfNhgOrch, gRouteOrch, copp_orch, qos_orch, wm_orch, policer_orch, tunnel_decap_orch, sflow_orch, debug_counter_orch, gMacsecOrch, gBfdOrch, gSrv6Orch}; bool initialize_dtel = false; if (platform == BFN_PLATFORM_SUBSTRING || platform == VS_PLATFORM_SUBSTRING) diff --git a/orchagent/orchdaemon.h b/orchagent/orchdaemon.h index 92527407a2..aca8c28697 100644 --- a/orchagent/orchdaemon.h +++ b/orchagent/orchdaemon.h @@ -12,7 +12,8 @@ #include "neighorch.h" #include "routeorch.h" #include "nhgorch.h" -#include "nhgmaporch.h" +#include "cbf/cbfnhgorch.h" +#include "cbf/nhgmaporch.h" #include "copporch.h" #include "tunneldecaporch.h" #include "qosorch.h" diff --git a/orchagent/qosorch.cpp b/orchagent/qosorch.cpp index fbb8c4a6b5..ae1e273634 100644 --- a/orchagent/qosorch.cpp +++ b/orchagent/qosorch.cpp @@ -3,7 +3,7 @@ #include "logger.h" #include "crmorch.h" #include "sai_serialize.h" -#include "nhgmaporch.h" +#include "cbf/nhgmaporch.h" #include #include diff --git a/tests/mock_tests/Makefile.am b/tests/mock_tests/Makefile.am index 8541fa5287..51bcf30547 100644 --- a/tests/mock_tests/Makefile.am +++ b/tests/mock_tests/Makefile.am @@ -39,10 +39,10 @@ tests_SOURCES = aclorch_ut.cpp \ $(top_srcdir)/orchagent/routeorch.cpp \ $(top_srcdir)/orchagent/mplsrouteorch.cpp \ $(top_srcdir)/orchagent/fgnhgorch.cpp \ - $(top_srcdir)/orchagent/nhg/nhgbase.cpp \ - $(top_srcdir)/orchagent/nhg/nhghandler.cpp \ - $(top_srcdir)/orchagent/nhg/cbfnhghandler.cpp \ - $(top_srcdir)/orchagent/nhgmaporch.cpp \ + $(top_srcdir)/orchagent/nhgbase.cpp \ + $(top_srcdir)/orchagent/nhgorch.cpp \ + $(top_srcdir)/orchagent/cbf/cbfnhgorch.cpp \ + $(top_srcdir)/orchagent/cbf/nhgmaporch.cpp \ $(top_srcdir)/orchagent/neighorch.cpp \ $(top_srcdir)/orchagent/intfsorch.cpp \ $(top_srcdir)/orchagent/portsorch.cpp \ From ae0d47ca1d7f3499db5f63395c8be1a4143c89c2 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 26 Oct 2021 09:59:15 +0100 Subject: [PATCH 06/29] Remove unneeded addition --- orchagent/nhgorch.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index e7cccf48fb..589a8d2166 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -588,7 +588,6 @@ bool NextHopGroup::sync() else { m_id = nhid; - gNeighOrch->increaseNextHopRefCount(nhgm.getKey()); } } else @@ -706,13 +705,7 @@ bool NextHopGroup::remove() // If the group is temporary, there is nothing to be done - just reset the ID. if (m_is_temp) { - if (m_id != SAI_NULL_OBJECT_ID) - { - m_id = SAI_NULL_OBJECT_ID; - - const NextHopGroupMember& nhgm = m_members.begin()->second; - gNeighOrch->decreaseNextHopRefCount(nhgm.getKey()); - } + m_id = SAI_NULL_OBJECT_ID; return true; } From 640d11b92239e21063cacb791fcc657b8caaeaa8 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 26 Oct 2021 14:27:34 +0100 Subject: [PATCH 07/29] Pyut tidy up --- tests/test_nhg.py | 2 +- tests/test_qos_map.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index b57940021d..cd59fe2ee9 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -93,7 +93,7 @@ def get_nhg_map_id(self, nhg_map_index): # If the CBF NHG can't be created, the provided NHG map index is invalid try: self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, asic_nhgs_count + 2) - except: + except Exception as e: # Remove the added NHGs cbf_nhg_ps._del('testcbfnhg') nhg_ps._del('testnhg') diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index 4faddc9fbf..c8eb3951b6 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -152,9 +152,6 @@ def test_dscp_to_fc(self, dvs): fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) assert(fvs.get("SAI_QOS_MAP_ATTR_TYPE") == "SAI_QOS_MAP_TYPE_DSCP_TO_FORWARDING_CLASS") - # Apply port configuration - # self.port_qos_map_profile_test('dscp_to_fc_map', swsscommon.CFG_DSCP_TO_FC_MAP_TABLE_NAME, 'SAI_PORT_ATTR_QOS_DSCP_TO_FORWARDING_CLASS_MAP') - # Delete the map self.dscp_ps._del("AZURE") self.asic_db.wait_for_deleted_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) From 74aae5cdc2611f8041987af2c99440436428cdd2 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 26 Oct 2021 16:10:13 +0100 Subject: [PATCH 08/29] Fix UT timing --- tests/test_qos_map.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index c8eb3951b6..39d4e11a42 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -141,6 +141,7 @@ def test_dscp_to_fc(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -183,6 +184,7 @@ def test_exp_to_fc(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -234,6 +236,7 @@ def test_per_port_cbf_binding(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) dscp_map_id = self.get_qos_id() self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) @@ -241,6 +244,7 @@ def test_per_port_cbf_binding(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 2) exp_map_id = self.get_qos_id() From f200dc0359477b9803c5afc3565eb62a9a557051 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 27 Oct 2021 14:27:28 +0100 Subject: [PATCH 09/29] Tweak UT timings --- tests/test_qos_map.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index 39d4e11a42..134099c27e 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -129,6 +129,7 @@ def init_test(self, dvs): # Set switch FC capability to 63 dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') + time.sleep(1) def get_qos_id(self): diff = set(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) - set(self.asic_qos_map_ids) @@ -141,7 +142,7 @@ def test_dscp_to_fc(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(1) + time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -184,7 +185,7 @@ def test_exp_to_fc(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) - time.sleep(1) + time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -236,7 +237,7 @@ def test_per_port_cbf_binding(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(1) + time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) dscp_map_id = self.get_qos_id() self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) From f92d577658e0f6cf872059cfb1306443a0187849 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 27 Oct 2021 15:39:04 +0100 Subject: [PATCH 10/29] Revert "Tweak UT timings" This reverts commit d5f40e8b124fbebb63717f546b6e27f4cececa17. --- tests/test_qos_map.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index 134099c27e..39d4e11a42 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -129,7 +129,6 @@ def init_test(self, dvs): # Set switch FC capability to 63 dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') - time.sleep(1) def get_qos_id(self): diff = set(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) - set(self.asic_qos_map_ids) @@ -142,7 +141,7 @@ def test_dscp_to_fc(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(2) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -185,7 +184,7 @@ def test_exp_to_fc(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) - time.sleep(2) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -237,7 +236,7 @@ def test_per_port_cbf_binding(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(2) + time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) dscp_map_id = self.get_qos_id() self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) From 9dadb68e24795a75c7c9eba2923c832af14b0d37 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 27 Oct 2021 14:27:28 +0100 Subject: [PATCH 11/29] Tweak UT timings --- tests/test_qos_map.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index 39d4e11a42..134099c27e 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -129,6 +129,7 @@ def init_test(self, dvs): # Set switch FC capability to 63 dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') + time.sleep(1) def get_qos_id(self): diff = set(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) - set(self.asic_qos_map_ids) @@ -141,7 +142,7 @@ def test_dscp_to_fc(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(1) + time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -184,7 +185,7 @@ def test_exp_to_fc(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) - time.sleep(1) + time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -236,7 +237,7 @@ def test_per_port_cbf_binding(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(1) + time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) dscp_map_id = self.get_qos_id() self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) From 463619e86b28676050838ae5e153a27c1926c2d3 Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Thu, 28 Oct 2021 16:55:35 +0300 Subject: [PATCH 12/29] Revert "Tweak UT timings" This reverts commit 40aa75c535438c6d30f2beb76146bae2c8032469. --- tests/test_qos_map.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index 134099c27e..c8eb3951b6 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -129,7 +129,6 @@ def init_test(self, dvs): # Set switch FC capability to 63 dvs.setReadOnlyAttr('SAI_OBJECT_TYPE_SWITCH', 'SAI_SWITCH_ATTR_MAX_NUMBER_OF_FORWARDING_CLASSES', '63') - time.sleep(1) def get_qos_id(self): diff = set(self.asic_db.get_keys(self.ASIC_QOS_MAP_STR)) - set(self.asic_qos_map_ids) @@ -142,7 +141,6 @@ def test_dscp_to_fc(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -185,7 +183,6 @@ def test_exp_to_fc(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) - time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) @@ -237,7 +234,6 @@ def test_per_port_cbf_binding(self, dvs): # Create a DSCP_TO_FC map dscp_map = [(str(i), str(i)) for i in range(0, 64)] self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) - time.sleep(2) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 1) dscp_map_id = self.get_qos_id() self.asic_qos_map_ids = self.asic_db.get_keys(self.ASIC_QOS_MAP_STR) @@ -245,7 +241,6 @@ def test_per_port_cbf_binding(self, dvs): # Create a EXP_TO_FC map exp_map = [(str(i), str(i)) for i in range(0, 8)] self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) - time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_QOS_MAP_STR, self.asic_qos_map_count + 2) exp_map_id = self.get_qos_id() From e119db41dc980bbce5903182577f50587bcc856e Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Fri, 29 Oct 2021 12:57:52 +0300 Subject: [PATCH 13/29] Split NhgOrch from CbfNhgOrch --- orchagent/cbf/cbfnhgorch.cpp | 8 ++-- orchagent/cbf/cbfnhgorch.h | 4 +- orchagent/mplsrouteorch.cpp | 66 ++++++++++++++++++++++++--------- orchagent/nhgbase.h | 25 ++++++++++++- orchagent/nhgorch.cpp | 43 +--------------------- orchagent/nhgorch.h | 49 +------------------------ orchagent/routeorch.cpp | 71 +++++++++++++++++++++++++++--------- 7 files changed, 135 insertions(+), 131 deletions(-) diff --git a/orchagent/cbf/cbfnhgorch.cpp b/orchagent/cbf/cbfnhgorch.cpp index 8b83ff7867..3c91d344eb 100644 --- a/orchagent/cbf/cbfnhgorch.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -9,6 +9,7 @@ extern sai_object_id_t gSwitchId; extern NhgOrch *gNhgOrch; +extern CbfNhgOrch *gCbfNhgOrch; extern CrmOrch *gCrmOrch; extern NhgMapOrch *gNhgMapOrch; extern RouteOrch *gRouteOrch; @@ -17,8 +18,7 @@ extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern size_t gMaxBulkSize; -CbfNhgOrch::CbfNhgOrch(DBConnector *db, string tableName) : - Orch(db, tableName) +CbfNhgOrch::CbfNhgOrch(DBConnector *db, string tableName) : NhgOrchCommon(db, tableName) { SWSS_LOG_ENTER(); } @@ -331,10 +331,10 @@ bool CbfNhg::sync() SWSS_LOG_ERROR("Failed to create CBF next hop group %s, rv %d", m_key.c_str(), status); - task_process_status handle_status = gNhgOrch->handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status); + task_process_status handle_status = gCbfNhgOrch->handleSaiCreateStatus(SAI_API_NEXT_HOP_GROUP, status); if (handle_status != task_success) { - return gNhgOrch->parseHandleSaiStatusFailure(handle_status); + return gCbfNhgOrch->parseHandleSaiStatusFailure(handle_status); } } diff --git a/orchagent/cbf/cbfnhgorch.h b/orchagent/cbf/cbfnhgorch.h index e4631cb9cb..9d13e94e73 100644 --- a/orchagent/cbf/cbfnhgorch.h +++ b/orchagent/cbf/cbfnhgorch.h @@ -85,7 +85,7 @@ class CbfNhg : public NhgCommon bool hasSameMembers(const vector &members) const; }; -class CbfNhgOrch : public Orch, public NhgOrchCommon +class CbfNhgOrch : public NhgOrchCommon { public: /* @@ -96,5 +96,5 @@ class CbfNhgOrch : public Orch, public NhgOrchCommon private: static pair> getMembers(const string &members); - void doTask(Consumer &consumer); + void doTask(Consumer &consumer) override; }; diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index 86b7eea75e..f67314ec65 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -8,12 +8,14 @@ #include "crmorch.h" #include "nhgorch.h" #include "directory.h" +#include "cbf/cbfnhgorch.h" extern sai_object_id_t gVirtualRouterId; extern sai_object_id_t gSwitchId; extern CrmOrch *gCrmOrch; extern NhgOrch *gNhgOrch; +extern CbfNhgOrch *gCbfNhgOrch; void RouteOrch::doLabelTask(Consumer& consumer) { @@ -251,18 +253,25 @@ void RouteOrch::doLabelTask(Consumer& consumer) } else { - try + const NhgBase *nh_group; + + if (gNhgOrch->hasNhg(nhg_index)) + { + nh_group = &gNhgOrch->getNhg(nhg_index); + } + else if (gCbfNhgOrch->hasNhg(nhg_index)) { - const auto &nh_group = gNhgOrch->getNhg(nhg_index); - ctx.nhg = nh_group.getNhgKey(); - ctx.using_temp_nhg = nh_group.isTemp(); + nh_group = &gCbfNhgOrch->getNhg(nhg_index); } - catch (const std::out_of_range& e) + else { SWSS_LOG_ERROR("Next hop group %s does not exist", nhg_index.c_str()); ++it; continue; } + + ctx.nhg = nh_group->getNhgKey(); + ctx.using_temp_nhg = nh_group->isTemp(); } NextHopGroupKey& nhg = ctx.nhg; @@ -308,7 +317,7 @@ void RouteOrch::doLabelTask(Consumer& consumer) // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount && + if (m_nextHopGroupCount + NhgOrch::getSyncedNhgCount() >= m_maxNextHopGroupCount && gLabelRouteBulker.removing_entries_count() > 0) { break; @@ -476,16 +485,23 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey if (!ctx.nhg_index.empty()) { - try + const NhgBase *nhg; + + if (gNhgOrch->hasNhg(ctx.nhg_index)) + { + nhg = &gNhgOrch->getNhg(ctx.nhg_index); + } + else if (gCbfNhgOrch->hasNhg(ctx.nhg_index)) { - const auto &nhg = gNhgOrch->getNhg(ctx.nhg_index); - next_hop_id = nhg.getId(); + nhg = &gCbfNhgOrch->getNhg(ctx.nhg_index); } - catch(const std::out_of_range& e) + else { SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); return false; } + + next_hop_id = nhg->getId(); } else if (nextHops.getSize() == 0) { @@ -673,7 +689,7 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo /* Check that the next hop group is not owned by NhgOrch. */ if (!ctx.nhg_index.empty()) { - if (!gNhgOrch->hasNhg(ctx.nhg_index)) + if (!gNhgOrch->hasNhg(ctx.nhg_index) && !gCbfNhgOrch->hasNhg(ctx.nhg_index)) { SWSS_LOG_WARN("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); return false; @@ -745,10 +761,14 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo { increaseNextHopRefCount(nextHops); } - else + else if (gNhgOrch->hasNhg(ctx.nhg_index)) { gNhgOrch->incNhgRefCount(ctx.nhg_index); } + else + { + gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + } SWSS_LOG_INFO("Post create label %u with next hop(s) %s", label, nextHops.to_string().c_str()); @@ -795,21 +815,29 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo m_bulkNhgReducedRefCnt.emplace(it_route->second.nhg_key, 0); } } - else + /* The next hop group is owned by (Cbf)NhgOrch. */ + else if (gNhgOrch->hasNhg(it_route->second.nhg_index)) { - /* The next hop group is owned by NeighOrch. */ gNhgOrch->decNhgRefCount(it_route->second.nhg_index); } + else + { + gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } /* Increase the ref_count for the next hop (group) entry */ if (ctx.nhg_index.empty()) { increaseNextHopRefCount(nextHops); } - else + else if (gNhgOrch->hasNhg(ctx.nhg_index)) { gNhgOrch->incNhgRefCount(ctx.nhg_index); } + else + { + gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + } if (blackhole) { @@ -928,10 +956,14 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) } } } - else + else if (gNhgOrch->hasNhg(it_route->second.nhg_index)) { gNhgOrch->decNhgRefCount(it_route->second.nhg_index); } + else + { + gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } SWSS_LOG_INFO("Remove label route %u with next hop(s) %s", label, it_route->second.nhg_key.to_string().c_str()); @@ -945,4 +977,4 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) } return true; -} \ No newline at end of file +} diff --git a/orchagent/nhgbase.h b/orchagent/nhgbase.h index f8cb6e1cfe..65f0690555 100644 --- a/orchagent/nhgbase.h +++ b/orchagent/nhgbase.h @@ -8,6 +8,7 @@ #include "set" #include "orch.h" #include "crmorch.h" +#include "routeorch.h" #include "nexthopgroupkey.h" #include "bulker.h" @@ -16,6 +17,7 @@ using namespace std; extern sai_object_id_t gSwitchId; extern CrmOrch *gCrmOrch; +extern RouteOrch *gRouteOrch; extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern size_t gMaxBulkSize; @@ -393,9 +395,14 @@ struct NhgEntry * Class providing the common functionality shared by all NhgOrch classes. */ template -class NhgOrchCommon +class NhgOrchCommon : public Orch { public: + /* + * Constructor. + */ + NhgOrchCommon(DBConnector *db, string tableName) : Orch(db, tableName) {} + /* * Check if the given next hop group index exists. */ @@ -433,6 +440,22 @@ class NhgOrchCommon --nhg_entry.ref_count; } + /* Getters / Setters. */ + static inline unsigned getSyncedNhgCount() { return NhgBase::getSyncedCount(); } + + /* Increase / Decrease the number of synced next hop groups. */ + inline void incSyncedNhgCount() + { + assert(gRouteOrch->getNhgCount() + NhgBase::getSyncedCount() < gRouteOrch->getMaxNhgCount()); + NhgBase::incSyncedCount(); + } + inline void decSyncedNhgCount() { NhgBase::decSyncedCount(); } + + /* Handling SAI status*/ + using Orch::handleSaiCreateStatus; + using Orch::handleSaiRemoveStatus; + using Orch::parseHandleSaiStatusFailure; + protected: /* * Map of synced next hop groups. diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 589a8d2166..87eec53dec 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -17,8 +17,7 @@ extern size_t gMaxBulkSize; extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern sai_next_hop_api_t* sai_next_hop_api; -NhgOrch::NhgOrch(DBConnector *db, string tableName) : - Orch(db, tableName) +NhgOrch::NhgOrch(DBConnector *db, string tableName) : NhgOrchCommon(db, tableName) { SWSS_LOG_ENTER(); } @@ -355,46 +354,6 @@ bool NhgOrch::invalidateNextHop(const NextHopKey& nh_key) return true; } -/* - * Purpose: Increase the ref count for a next hop group. - * Description: Increment the ref count for a next hop group by 1. - * Params: IN index - The index of the next hop group. - * Returns: Nothing. - */ -void NhgOrch::incNhgRefCount(const std::string& index) -{ - SWSS_LOG_ENTER(); - - try - { - NhgOrchCommon::incNhgRefCount(index); - } - catch(const std::out_of_range &e) - { - gCbfNhgOrch->incNhgRefCount(index); - } -} - -/* - * Purpose: Decrease the ref count for a next hop group. - * Description: Decrement the ref count for a next hop group by 1. - * Params: IN index - The index of the next hop group. - * Returns: Nothing. - */ -void NhgOrch::decNhgRefCount(const std::string& index) -{ - SWSS_LOG_ENTER(); - - try - { - NhgOrchCommon::decNhgRefCount(index); - } - catch(const std::out_of_range &e) - { - gCbfNhgOrch->decNhgRefCount(index); - } -} - /* * Purpose: Get the next hop ID of the member. * Description: Get the SAI ID of the next hop from NeighOrch. diff --git a/orchagent/nhgorch.h b/orchagent/nhgorch.h index 4c4d5fa456..225d3ffaf2 100644 --- a/orchagent/nhgorch.h +++ b/orchagent/nhgorch.h @@ -9,7 +9,6 @@ using namespace std; extern PortsOrch *gPortsOrch; extern RouteOrch *gRouteOrch; -extern CbfNhgOrch *gCbfNhgOrch; class NextHopGroupMember : public NhgMember { @@ -108,7 +107,7 @@ class NextHopGroup : public NhgCommon, public Orch +class NhgOrch : public NhgOrchCommon { public: /* @@ -116,57 +115,13 @@ class NhgOrch : public NhgOrchCommon, public Orch */ NhgOrch(DBConnector *db, string tableName); - /* Check if the next hop group given by it's index exists. */ - inline bool hasNhg(const std::string& index) const - { - return NhgOrchCommon::hasNhg(index) || gCbfNhgOrch->hasNhg(index); - } - - /* - * Get the next hop group with the given index. - */ - inline const NhgBase& getNhg(const std::string &index) const - { - try - { - return NhgOrchCommon::getNhg(index); - } - catch(const std::out_of_range &e) - { - return gCbfNhgOrch->getNhg(index); - } - } - /* Add a temporary next hop group when resources are exhausted. */ NextHopGroup createTempNhg(const NextHopGroupKey& nhg_key); - /* Getters / Setters. */ - static inline unsigned getSyncedNhgCount() { return NhgBase::getSyncedCount(); } - - /* Increase / Decrease the number of synced next hop groups. */ - inline void incSyncedNhgCount() - { - assert(gRouteOrch->getNhgCount() + NhgBase::getSyncedCount() < gRouteOrch->getMaxNhgCount()); - NhgBase::incSyncedCount(); - } - inline void decSyncedNhgCount() { NhgBase::decSyncedCount(); } - - /* Increase / Decrease ref count for a NHG given by it's index. */ - void incNhgRefCount(const std::string& index); - void decNhgRefCount(const std::string& index); - /* Validate / Invalidate a next hop. */ bool validateNextHop(const NextHopKey& nh_key); bool invalidateNextHop(const NextHopKey& nh_key); - /* Handling SAI status*/ - task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, void *context = nullptr) - { return Orch::handleSaiCreateStatus(api, status, context); } - task_process_status handleSaiRemoveStatus(sai_api_t api, sai_status_t status, void *context = nullptr) - { return Orch::handleSaiRemoveStatus(api, status, context); } - bool parseHandleSaiStatusFailure(task_process_status status) - { return Orch::parseHandleSaiStatusFailure(status); } private: - - void doTask(Consumer& consumer); + void doTask(Consumer& consumer) override; }; diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index dd70891a10..cf0b72e672 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -3,6 +3,7 @@ #include #include "routeorch.h" #include "nhgorch.h" +#include "cbf/cbfnhgorch.h" #include "logger.h" #include "swssnet.h" #include "crmorch.h" @@ -20,6 +21,7 @@ extern PortsOrch *gPortsOrch; extern CrmOrch *gCrmOrch; extern Directory gDirectory; extern NhgOrch *gNhgOrch; +extern CbfNhgOrch *gCbfNhgOrch; extern size_t gMaxBulkSize; @@ -759,18 +761,25 @@ void RouteOrch::doTask(Consumer& consumer) } else { - try + const NhgBase *nh_group; + + if (gNhgOrch->hasNhg(nhg_index)) + { + nh_group = &gNhgOrch->getNhg(nhg_index); + } + else if (gCbfNhgOrch->hasNhg(nhg_index)) { - const auto &nh_group = gNhgOrch->getNhg(nhg_index); - nhg = nh_group.getNhgKey(); - ctx.using_temp_nhg = nh_group.isTemp(); + nh_group = &gCbfNhgOrch->getNhg(nhg_index); } - catch (const std::out_of_range& e) + else { SWSS_LOG_ERROR("Next hop group %s does not exist", nhg_index.c_str()); ++it; continue; } + + nhg = nh_group->getNhgKey(); + ctx.using_temp_nhg = nh_group->isTemp(); } if (nhg.getSize() == 1 && nhg.hasIntfNextHop()) @@ -831,7 +840,7 @@ void RouteOrch::doTask(Consumer& consumer) // If already exhaust the nexthop groups, and there are pending removing routes in bulker, // flush the bulker and possibly collect some released nexthop groups - if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount && + if (m_nextHopGroupCount + NhgOrch::getSyncedNhgCount() >= m_maxNextHopGroupCount && gRouteBulker.removing_entries_count() > 0) { break; @@ -1107,7 +1116,7 @@ bool RouteOrch::createFineGrainedNextHopGroup(sai_object_id_t &next_hop_group_id { SWSS_LOG_ENTER(); - if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount) + if (m_nextHopGroupCount + NhgOrch::getSyncedNhgCount() >= m_maxNextHopGroupCount) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1162,7 +1171,7 @@ bool RouteOrch::addNextHopGroup(const NextHopGroupKey &nexthops) assert(!hasNextHopGroup(nexthops)); - if (m_nextHopGroupCount + gNhgOrch->getSyncedNhgCount() >= m_maxNextHopGroupCount) + if (m_nextHopGroupCount + NhgOrch::getSyncedNhgCount() >= m_maxNextHopGroupCount) { SWSS_LOG_DEBUG("Failed to create new next hop group. \ Reaching maximum number of next hop groups."); @@ -1628,16 +1637,23 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) /* NhgOrch owns the NHG */ else if (!ctx.nhg_index.empty()) { - try + const NhgBase *nhg; + + if (gNhgOrch->hasNhg(ctx.nhg_index)) + { + nhg = &gNhgOrch->getNhg(ctx.nhg_index); + } + else if (gCbfNhgOrch->hasNhg(ctx.nhg_index)) { - const auto &nhg = gNhgOrch->getNhg(ctx.nhg_index); - next_hop_id = nhg.getId(); + nhg = &gCbfNhgOrch->getNhg(ctx.nhg_index); } - catch(const std::out_of_range& e) + else { SWSS_LOG_INFO("Next hop group key %s does not exist", ctx.nhg_index.c_str()); return false; } + + next_hop_id = nhg->getId(); } /* RouteOrch owns the NHG */ else if (nextHops.getSize() == 0) @@ -1898,7 +1914,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* NhgOrch owns the NHG. */ else if (!ctx.nhg_index.empty()) { - if (!gNhgOrch->hasNhg(ctx.nhg_index)) + if (!gNhgOrch->hasNhg(ctx.nhg_index) && !gCbfNhgOrch->hasNhg(ctx.nhg_index)) { SWSS_LOG_INFO("Failed to get next hop group with index %s", ctx.nhg_index.c_str()); return false; @@ -2028,10 +2044,14 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey { increaseNextHopRefCount(nextHops); } - else + else if (gNhgOrch->hasNhg(ctx.nhg_index)) { gNhgOrch->incNhgRefCount(ctx.nhg_index); } + else + { + gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + } SWSS_LOG_INFO("Post create route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); @@ -2099,11 +2119,15 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey removeNextHopRoute(nexthop, r_key); } } - else + /* The next hop group is owned by (Cbf)NhgOrch. */ + else if (gNhgOrch->hasNhg(it_route->second.nhg_index)) { - /* The next hop group is owned by NhgOrch. */ gNhgOrch->decNhgRefCount(it_route->second.nhg_index); } + else + { + gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } if (blackhole) { @@ -2126,10 +2150,14 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Increase the ref_count for the next hop (group) entry */ increaseNextHopRefCount(nextHops); } - else + else if (gNhgOrch->hasNhg(ctx.nhg_index)) { gNhgOrch->incNhgRefCount(ctx.nhg_index); } + else + { + gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + } SWSS_LOG_INFO("Post set route %s with next hop(s) %s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); @@ -2293,7 +2321,14 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) /* Check if the next hop group is not owned by NhgOrch. */ else if (!it_route->second.nhg_index.empty()) { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + if (gNhgOrch->hasNhg(it_route->second.nhg_index)) + { + gNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } + else + { + gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + } } /* The NHG is owned by RouteOrch */ else From bda9c9c090af5ed5cf7fd4a54426c8543c01bc95 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Mon, 1 Nov 2021 14:58:21 +0000 Subject: [PATCH 14/29] Fix weighted ECMP members --- orchagent/nhgorch.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index 87eec53dec..b29010f25f 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -866,10 +866,14 @@ vector NextHopGroup::createNhgmAttrs(const NextHopGroupMember& nhgm_attr.value.oid = nhgm.getNhId(); nhgm_attrs.push_back(nhgm_attr); - /* Fill in the wright. */ - nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; - nhgm_attr.value.s32 = nhgm.getWeight(); - nhgm_attrs.push_back(nhgm_attr); + /* Fill in the weight if set. */ + auto weight = nhgm.getWeight(); + if (weight != 0) + { + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT; + nhgm_attr.value.s32 = weight; + nhgm_attrs.push_back(nhgm_attr); + } return nhgm_attrs; } From e4f57ce1f954631ec80970f7b3c9a2b13e7207ad Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 3 Nov 2021 16:57:06 +0000 Subject: [PATCH 15/29] Fix UT --- tests/test_nhg.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index cd59fe2ee9..aac0bd1188 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -1173,14 +1173,6 @@ def mainline_labeled_nhs_test(): self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 2) self.asic_db.wait_for_n_keys(self.ASIC_NHS_STR, self.asic_nhs_count + 1) - # Assert the weights of the NHGMs are the expected ones - nhgm_ids = self.get_nhgm_ids('group1') - weights = [] - for nhgm_id in nhgm_ids: - fvs = self.asic_db.get_entry(self.ASIC_NHGM_STR, nhgm_id) - weights.append(fvs['SAI_NEXT_HOP_GROUP_MEMBER_ATTR_WEIGHT']) - assert weights == ['0', '0'] - # Delete group1 self.nhg_ps._del('group1') From 93d726b3ef6c6c3da59174f15dc55a4c36b21d2f Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 4 Nov 2021 15:55:04 +0000 Subject: [PATCH 16/29] Fixes --- orchagent/cbf/cbfnhgorch.cpp | 15 +++++- orchagent/cbf/nhgmaporch.cpp | 26 +++++++-- orchagent/cbf/nhgmaporch.h | 6 ++- orchagent/mplsrouteorch.cpp | 56 +++++-------------- orchagent/routeorch.cpp | 101 +++++++++++++++++++---------------- orchagent/routeorch.h | 6 +++ 6 files changed, 116 insertions(+), 94 deletions(-) diff --git a/orchagent/cbf/cbfnhgorch.cpp b/orchagent/cbf/cbfnhgorch.cpp index 3c91d344eb..5261792921 100644 --- a/orchagent/cbf/cbfnhgorch.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -314,7 +314,13 @@ bool CbfNhg::sync() if (nhg_attr.value.oid == SAI_NULL_OBJECT_ID) { - SWSS_LOG_WARN("FC to NHG map index %s does not exist", m_selection_map.c_str()); + SWSS_LOG_ERROR("FC to NHG map index %s does not exist", m_selection_map.c_str()); + return false; + } + + if ((unsigned int)gNhgMapOrch->getLargestNhIndex(m_selection_map) > m_members.size()) + { + SWSS_LOG_ERROR("FC to NHG map references more NHG members than exist in group %s", m_key.c_str()); return false; } @@ -525,6 +531,13 @@ bool CbfNhg::update(const vector &members, const string &selection_map) m_members.emplace(member, CbfNhgMember(member, index++)); } + if ((unsigned int)gNhgMapOrch->getLargestNhIndex(m_selection_map) > m_members.size()) + { + SWSS_LOG_ERROR("FC to NHG map references more NHG members than exist in group %s", + m_key.c_str()); + return false; + } + /* Sync the new members. */ if (!syncMembers({members.begin(), members.end()})) { diff --git a/orchagent/cbf/nhgmaporch.cpp b/orchagent/cbf/nhgmaporch.cpp index 61d5ab33d1..318529321d 100644 --- a/orchagent/cbf/nhgmaporch.cpp +++ b/orchagent/cbf/nhgmaporch.cpp @@ -7,7 +7,8 @@ extern sai_switch_api_t *sai_switch_api; uint64_t NhgMapOrch::m_max_nhg_map_count = 0; -NhgMapEntry::NhgMapEntry(sai_object_id_t _id, uint32_t _ref_count) : id(_id), ref_count(_ref_count) +NhgMapEntry::NhgMapEntry(sai_object_id_t _id, uint32_t _ref_count, int _largest_nh_index) : + id(_id), ref_count(_ref_count), largest_nh_index(_largest_nh_index) { SWSS_LOG_ENTER(); } @@ -68,15 +69,21 @@ void NhgMapOrch::doTask(Consumer &consumer) else { /* - * Create the SAI map. + * Create the SAI map. Also track the largest index referenced by the map as we do. */ auto *fc_map = new sai_map_t[p.second.size()]; uint32_t ii = 0; + int largest_nh_index = 0; for (const auto &fc_nh_idx : p.second) { fc_map[ii].key = fc_nh_idx.first; fc_map[ii++].value = fc_nh_idx.second; + + if (fc_nh_idx.second > largest_nh_index) + { + largest_nh_index = fc_nh_idx.second; + } } sai_map_list_t fc_map_list; @@ -130,7 +137,8 @@ void NhgMapOrch::doTask(Consumer &consumer) else { assert(nhg_map_id != SAI_NULL_OBJECT_ID); - m_syncdMaps.emplace(move(index), nhg_map_id); + NhgMapEntry entry(nhg_map_id, 0, largest_nh_index); + m_syncdMaps.emplace(move(index), entry); } } } @@ -239,6 +247,18 @@ sai_object_id_t NhgMapOrch::getMapId(const string &index) const return it == m_syncdMaps.end() ? SAI_NULL_OBJECT_ID : it->second.id; } +/* + * Return the largest NH index used by the map indexed by "index". If it does not exist, return 0. + */ +int NhgMapOrch::getLargestNhIndex(const string &index) const +{ + SWSS_LOG_ENTER(); + + auto it = m_syncdMaps.find(index); + + return it == m_syncdMaps.end() ? 0 : it->second.largest_nh_index; +} + /* * Increase reference counter for a map. */ diff --git a/orchagent/cbf/nhgmaporch.h b/orchagent/cbf/nhgmaporch.h index 8e5d2b976a..c345e7d566 100644 --- a/orchagent/cbf/nhgmaporch.h +++ b/orchagent/cbf/nhgmaporch.h @@ -17,7 +17,10 @@ struct NhgMapEntry /* Number of external objects referencing this next hop group map. */ uint32_t ref_count; - explicit NhgMapEntry(sai_object_id_t id, uint32_t _ref_count = 0); + /* The largest group index referenced by the map. */ + int largest_nh_index; + + explicit NhgMapEntry(sai_object_id_t id, uint32_t _ref_count, int _largest_nh_index); }; class NhgMapOrch : public Orch @@ -31,6 +34,7 @@ class NhgMapOrch : public Orch * Return the SAI ID for the map indexed by "index". If it does not exist, return SAI_NULL_OBJECT_ID. */ sai_object_id_t getMapId(const string &key) const; + int getLargestNhIndex(const string &key) const; /* * Increase / Decrease reference counter for a map. diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index f67314ec65..beb0a33688 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -253,25 +253,18 @@ void RouteOrch::doLabelTask(Consumer& consumer) } else { - const NhgBase *nh_group; - - if (gNhgOrch->hasNhg(nhg_index)) + try { - nh_group = &gNhgOrch->getNhg(nhg_index); + const NhgBase& nh_group = getNhg(nhg_index); + ctx.nhg = nh_group.getNhgKey(); + ctx.using_temp_nhg = nh_group.isTemp(); } - else if (gCbfNhgOrch->hasNhg(nhg_index)) - { - nh_group = &gCbfNhgOrch->getNhg(nhg_index); - } - else + catch (const std::out_of_range& e) { SWSS_LOG_ERROR("Next hop group %s does not exist", nhg_index.c_str()); ++it; continue; } - - ctx.nhg = nh_group->getNhgKey(); - ctx.using_temp_nhg = nh_group->isTemp(); } NextHopGroupKey& nhg = ctx.nhg; @@ -485,23 +478,16 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey if (!ctx.nhg_index.empty()) { - const NhgBase *nhg; - - if (gNhgOrch->hasNhg(ctx.nhg_index)) - { - nhg = &gNhgOrch->getNhg(ctx.nhg_index); - } - else if (gCbfNhgOrch->hasNhg(ctx.nhg_index)) + try { - nhg = &gCbfNhgOrch->getNhg(ctx.nhg_index); + const NhgBase& nhg = getNhg(ctx.nhg_index); + next_hop_id = nhg.getId(); } - else + catch(const std::out_of_range& e) { SWSS_LOG_WARN("Next hop group key %s does not exist", ctx.nhg_index.c_str()); return false; } - - next_hop_id = nhg->getId(); } else if (nextHops.getSize() == 0) { @@ -761,13 +747,9 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo { increaseNextHopRefCount(nextHops); } - else if (gNhgOrch->hasNhg(ctx.nhg_index)) - { - gNhgOrch->incNhgRefCount(ctx.nhg_index); - } else { - gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + incNhgRefCount(ctx.nhg_index); } SWSS_LOG_INFO("Post create label %u with next hop(s) %s", @@ -816,13 +798,9 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo } } /* The next hop group is owned by (Cbf)NhgOrch. */ - else if (gNhgOrch->hasNhg(it_route->second.nhg_index)) - { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); - } else { - gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + decNhgRefCount(it_route->second.nhg_index); } /* Increase the ref_count for the next hop (group) entry */ @@ -830,13 +808,9 @@ bool RouteOrch::addLabelRoutePost(const LabelRouteBulkContext& ctx, const NextHo { increaseNextHopRefCount(nextHops); } - else if (gNhgOrch->hasNhg(ctx.nhg_index)) - { - gNhgOrch->incNhgRefCount(ctx.nhg_index); - } else { - gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + incNhgRefCount(ctx.nhg_index); } if (blackhole) @@ -956,13 +930,9 @@ bool RouteOrch::removeLabelRoutePost(const LabelRouteBulkContext& ctx) } } } - else if (gNhgOrch->hasNhg(it_route->second.nhg_index)) - { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); - } else { - gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + decNhgRefCount(it_route->second.nhg_index); } SWSS_LOG_INFO("Remove label route %u with next hop(s) %s", diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index cf0b72e672..8337e6cba1 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -761,25 +761,18 @@ void RouteOrch::doTask(Consumer& consumer) } else { - const NhgBase *nh_group; - - if (gNhgOrch->hasNhg(nhg_index)) - { - nh_group = &gNhgOrch->getNhg(nhg_index); - } - else if (gCbfNhgOrch->hasNhg(nhg_index)) + try { - nh_group = &gCbfNhgOrch->getNhg(nhg_index); + const NhgBase& nh_group = getNhg(nhg_index); + nhg = nh_group.getNhgKey(); + ctx.using_temp_nhg = nh_group.isTemp(); } - else + catch (const std::out_of_range& e) { SWSS_LOG_ERROR("Next hop group %s does not exist", nhg_index.c_str()); ++it; continue; } - - nhg = nh_group->getNhgKey(); - ctx.using_temp_nhg = nh_group->isTemp(); } if (nhg.getSize() == 1 && nhg.hasIntfNextHop()) @@ -1637,23 +1630,16 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) /* NhgOrch owns the NHG */ else if (!ctx.nhg_index.empty()) { - const NhgBase *nhg; - - if (gNhgOrch->hasNhg(ctx.nhg_index)) + try { - nhg = &gNhgOrch->getNhg(ctx.nhg_index); + const NhgBase& nhg = getNhg(ctx.nhg_index); + next_hop_id = nhg.getId(); } - else if (gCbfNhgOrch->hasNhg(ctx.nhg_index)) - { - nhg = &gCbfNhgOrch->getNhg(ctx.nhg_index); - } - else + catch(const std::out_of_range& e) { SWSS_LOG_INFO("Next hop group key %s does not exist", ctx.nhg_index.c_str()); return false; } - - next_hop_id = nhg->getId(); } /* RouteOrch owns the NHG */ else if (nextHops.getSize() == 0) @@ -2044,13 +2030,9 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey { increaseNextHopRefCount(nextHops); } - else if (gNhgOrch->hasNhg(ctx.nhg_index)) - { - gNhgOrch->incNhgRefCount(ctx.nhg_index); - } else { - gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + incNhgRefCount(ctx.nhg_index); } SWSS_LOG_INFO("Post create route %s with next hop(s) %s", @@ -2120,13 +2102,9 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey } } /* The next hop group is owned by (Cbf)NhgOrch. */ - else if (gNhgOrch->hasNhg(it_route->second.nhg_index)) - { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); - } else { - gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); + decNhgRefCount(it_route->second.nhg_index); } if (blackhole) @@ -2150,13 +2128,9 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Increase the ref_count for the next hop (group) entry */ increaseNextHopRefCount(nextHops); } - else if (gNhgOrch->hasNhg(ctx.nhg_index)) - { - gNhgOrch->incNhgRefCount(ctx.nhg_index); - } else { - gCbfNhgOrch->incNhgRefCount(ctx.nhg_index); + incNhgRefCount(ctx.nhg_index); } SWSS_LOG_INFO("Post set route %s with next hop(s) %s", @@ -2321,14 +2295,7 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) /* Check if the next hop group is not owned by NhgOrch. */ else if (!it_route->second.nhg_index.empty()) { - if (gNhgOrch->hasNhg(it_route->second.nhg_index)) - { - gNhgOrch->decNhgRefCount(it_route->second.nhg_index); - } - else - { - gCbfNhgOrch->decNhgRefCount(it_route->second.nhg_index); - } + decNhgRefCount(it_route->second.nhg_index); } /* The NHG is owned by RouteOrch */ else @@ -2487,3 +2454,45 @@ bool RouteOrch::checkNextHopGroupCount() { return m_nextHopGroupCount < m_maxNextHopGroupCount; } + +const NhgBase &RouteOrch::getNhg(const std::string &nhg_index) +{ + SWSS_LOG_ENTER(); + + try + { + return gNhgOrch->getNhg(nhg_index); + } + catch (const std::out_of_range& e) + { + return gCbfNhgOrch->getNhg(nhg_index); + } +} + +void RouteOrch::incNhgRefCount(const std::string &nhg_index) +{ + SWSS_LOG_ENTER(); + + if (gNhgOrch->hasNhg(nhg_index)) + { + gNhgOrch->incNhgRefCount(nhg_index); + } + else + { + gCbfNhgOrch->incNhgRefCount(nhg_index); + } +} + +void RouteOrch::decNhgRefCount(const std::string &nhg_index) +{ + SWSS_LOG_ENTER(); + + if (gNhgOrch->hasNhg(nhg_index)) + { + gNhgOrch->decNhgRefCount(nhg_index); + } + else + { + gCbfNhgOrch->decNhgRefCount(nhg_index); + } +} diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index 60af5c756a..74cd4c4442 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -26,6 +26,8 @@ typedef std::map NextHopGroupMembers; +struct NhgBase; + struct NextHopGroupEntry { sai_object_id_t next_hop_group_id; // next hop group id @@ -251,6 +253,10 @@ class RouteOrch : public Orch, public Subject void doTask(Consumer& consumer); void doLabelTask(Consumer& consumer); + + const NhgBase &getNhg(const std::string& nhg_index); + void incNhgRefCount(const std::string& nhg_index); + void decNhgRefCount(const std::string& nhg_index); }; #endif /* SWSS_ROUTEORCH_H */ From 719c6a1e15b0de88d34c709567a0b32d1a279ff1 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 4 Nov 2021 15:57:11 +0000 Subject: [PATCH 17/29] Fixes --- orchagent/cbf/cbfnhgorch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/cbf/cbfnhgorch.cpp b/orchagent/cbf/cbfnhgorch.cpp index 5261792921..902f5f9dbf 100644 --- a/orchagent/cbf/cbfnhgorch.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -556,7 +556,7 @@ bool CbfNhg::update(const vector &members, const string &selection_map) if (nhg_attr.value.oid == SAI_NULL_OBJECT_ID) { - SWSS_LOG_WARN("NHG map %s does not exist", selection_map.c_str()); + SWSS_LOG_ERROR("NHG map %s does not exist", selection_map.c_str()); return false; } From dd4eeb5bcfb4ff4737be76d1c59280764fa8e73e Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 4 Nov 2021 17:13:12 +0000 Subject: [PATCH 18/29] Fix and UT --- orchagent/cbf/cbfnhgorch.cpp | 11 ++++++++-- tests/test_nhg.py | 41 ++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/orchagent/cbf/cbfnhgorch.cpp b/orchagent/cbf/cbfnhgorch.cpp index 902f5f9dbf..52d57409ce 100644 --- a/orchagent/cbf/cbfnhgorch.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -318,7 +318,7 @@ bool CbfNhg::sync() return false; } - if ((unsigned int)gNhgMapOrch->getLargestNhIndex(m_selection_map) > m_members.size()) + if ((unsigned int)gNhgMapOrch->getLargestNhIndex(m_selection_map) >= m_members.size()) { SWSS_LOG_ERROR("FC to NHG map references more NHG members than exist in group %s", m_key.c_str()); return false; @@ -531,7 +531,7 @@ bool CbfNhg::update(const vector &members, const string &selection_map) m_members.emplace(member, CbfNhgMember(member, index++)); } - if ((unsigned int)gNhgMapOrch->getLargestNhIndex(m_selection_map) > m_members.size()) + if ((unsigned int)gNhgMapOrch->getLargestNhIndex(m_selection_map) >= m_members.size()) { SWSS_LOG_ERROR("FC to NHG map references more NHG members than exist in group %s", m_key.c_str()); @@ -560,6 +560,13 @@ bool CbfNhg::update(const vector &members, const string &selection_map) return false; } + if ((unsigned int)gNhgMapOrch->getLargestNhIndex(selection_map) >= m_members.size()) + { + SWSS_LOG_ERROR("FC to NHG map references more NHG members than exist in group %s", + m_key.c_str()); + return false; + } + auto status = sai_next_hop_group_api->set_next_hop_group_attribute(m_id, &nhg_attr); if (status != SAI_STATUS_SUCCESS) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index aac0bd1188..8e4073eb2e 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -1814,17 +1814,10 @@ def update_deleting_cbf_nhg_test(): nhgm_ids = self.get_nhgm_ids('cbfgroup1') nhg_id = self.get_nhg_id('cbfgroup1') old_map = self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] - fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap1')]) + fvs = swsscommon.FieldValuePairs([('members', 'group2,group3'), ('selection_map', 'cbfnhgmap1')]) self.cbf_nhg_ps.set('cbfgroup1', fvs) self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) - self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 4) - nhgm_id = self.get_nhgm_ids('cbfgroup1')[0] - self.asic_db.wait_for_field_match(self.ASIC_NHGM_STR, - nhgm_id, - {'SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX': '0'}) - self.asic_db.wait_for_field_negative_match(self.ASIC_NHG_STR, - nhg_id, - {'SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP': old_map}) + self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 5) # Delete the route self.rt_ps._del('2.2.2.0/24') @@ -1836,9 +1829,14 @@ def update_deleting_cbf_nhg_test(): # - try updating the CBF NHG with a member that doesn't exist and assert the CBF NHG's # - create the missing NHG and assert the CBF NHG's member also gets created def update_cbf_nhg_inexistent_member_test(): + # Create an FC to NH index selection map that references just 1 group member + fvs = swsscommon.FieldValuePairs([('0', '0')]) + self.fc_to_nhg_ps.set('cbfnhgmap4', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 3) + # Update the CBF NHG referencing an NHG that doesn't exist. In the end, create the NHG and # make sure everything works fine. - fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap1')]) + fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap4')]) self.cbf_nhg_ps.set('cbfgroup1', fvs) time.sleep(1) self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) @@ -1860,11 +1858,23 @@ def update_cbf_nhg_inexistent_map_test(): self.cbf_nhg_ps.set('cbfgroup1', fvs) time.sleep(1) assert(self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] == smap_id) - fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap2')]) + fvs = swsscommon.FieldValuePairs([('members', 'group2'), ('selection_map', 'cbfnhgmap4')]) self.cbf_nhg_ps.set('cbfgroup1', fvs) - self.asic_db.wait_for_field_negative_match(self.ASIC_NHG_STR, - nhg_id, - {'SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP': smap_id}) + + # Test scenario: + # - create a NHG that points to a map that refers to more members than the group has + def create_cbf_invalid_nhg_map_test(): + # Create an FC to NH index selection map that references 3 group members + fvs = swsscommon.FieldValuePairs([('0', '1'), ('1', '0'), ('2', '2')]) + self.fc_to_nhg_ps.set('cbfnhgmap3', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count + 2) + + # Create a group that references this map. It doesn't get created. + fvs = swsscommon.FieldValuePairs([('members', 'group3,group2'), + ('selection_map', 'cbfnhgmap3')]) + self.cbf_nhg_ps.set('cbfgroup3', fvs) + time.sleep(1) + assert(not self.nhg_exists('cbfgroup3')) self.init_test(dvs, 4) @@ -1875,6 +1885,7 @@ def update_cbf_nhg_inexistent_map_test(): delete_referenced_cbf_nhg_test() create_route_inexistent_cbf_nhg_test() update_deleting_cbf_nhg_test() + create_cbf_invalid_nhg_map_test() # Delete the NHGs self.cbf_nhg_ps._del('cbfgroup1') @@ -1899,6 +1910,8 @@ def update_cbf_nhg_inexistent_map_test(): # Delete the NHG maps self.fc_to_nhg_ps._del('cbfnhgmap1') self.fc_to_nhg_ps._del('cbfnhgmap2') + self.fc_to_nhg_ps._del('cbfnhgmap3') + self.fc_to_nhg_ps._del('cbfnhgmap4') self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count) # Coverage testing: Delete inexistent CBF NHG From 991120f4050936843a5b958c882286d6779c16ef Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Mon, 8 Nov 2021 11:39:57 +0000 Subject: [PATCH 19/29] CRM and warning log --- orchagent/cbf/cbfnhgorch.cpp | 6 ++++++ orchagent/cbf/nhgmaporch.cpp | 6 ++++++ orchagent/crmorch.cpp | 8 ++++++++ orchagent/crmorch.h | 1 + 4 files changed, 21 insertions(+) diff --git a/orchagent/cbf/cbfnhgorch.cpp b/orchagent/cbf/cbfnhgorch.cpp index 52d57409ce..4ccbe7cda6 100644 --- a/orchagent/cbf/cbfnhgorch.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -308,6 +308,12 @@ bool CbfNhg::sync() nhg_attr.value.u32 = static_cast(m_members.size()); nhg_attrs.push_back(move(nhg_attr)); + if (nhg_attr.value.u32 > gNhgMapOrch->getMaxFcVal) + { + /* If there are more members than FCs then this may be an error, as some members won't be used. */ + SWSS_LOG_WARN("More CBF NHG members configured than supported Forwarding Classes"); + } + /* Add the selection map to the attributes. */ nhg_attr.id = SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP; nhg_attr.value.oid = gNhgMapOrch->getMapId(m_selection_map); diff --git a/orchagent/cbf/nhgmaporch.cpp b/orchagent/cbf/nhgmaporch.cpp index 318529321d..d765e3e90e 100644 --- a/orchagent/cbf/nhgmaporch.cpp +++ b/orchagent/cbf/nhgmaporch.cpp @@ -1,12 +1,15 @@ #include "nhgmaporch.h" #include "climits" +#include "crmorch.h" extern sai_object_id_t gSwitchId; extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern sai_switch_api_t *sai_switch_api; +extern CrmOrch *gCrmOrch; uint64_t NhgMapOrch::m_max_nhg_map_count = 0; + NhgMapEntry::NhgMapEntry(sai_object_id_t _id, uint32_t _ref_count, int _largest_nh_index) : id(_id), ref_count(_ref_count), largest_nh_index(_largest_nh_index) { @@ -139,6 +142,8 @@ void NhgMapOrch::doTask(Consumer &consumer) assert(nhg_map_id != SAI_NULL_OBJECT_ID); NhgMapEntry entry(nhg_map_id, 0, largest_nh_index); m_syncdMaps.emplace(move(index), entry); + + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MAP); } } } @@ -203,6 +208,7 @@ void NhgMapOrch::doTask(Consumer &consumer) if (status == SAI_STATUS_SUCCESS) { m_syncdMaps.erase(fc_map_it); + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MAP); } else { diff --git a/orchagent/crmorch.cpp b/orchagent/crmorch.cpp index 8c651fffc7..fbc9d43691 100644 --- a/orchagent/crmorch.cpp +++ b/orchagent/crmorch.cpp @@ -45,6 +45,7 @@ const map crmResTypeNameMap = { CrmResourceType::CRM_MPLS_NEXTHOP, "MPLS_NEXTHOP" }, { CrmResourceType::CRM_SRV6_MY_SID_ENTRY, "SRV6_MY_SID_ENTRY" }, { CrmResourceType::CRM_SRV6_NEXTHOP, "SRV6_NEXTHOP" }, + { CrmResourceType::CRM_NEXTHOP_GROUP_MAP, "NEXTHOP_GROUP_MAP" }, }; const map crmResSaiAvailAttrMap = @@ -69,6 +70,7 @@ const map crmResSaiAvailAttrMap = { CrmResourceType::CRM_MPLS_NEXTHOP, SAI_SWITCH_ATTR_AVAILABLE_IPV4_NEXTHOP_ENTRY }, { CrmResourceType::CRM_SRV6_MY_SID_ENTRY, SAI_OBJECT_TYPE_MY_SID_ENTRY }, { CrmResourceType::CRM_SRV6_NEXTHOP, SAI_SWITCH_ATTR_AVAILABLE_IPV6_NEXTHOP_ENTRY }, + { CrmResourceType::CRM_NEXTHOP_GROUP_MAP, SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MAP }, }; const map crmThreshTypeResMap = @@ -93,6 +95,7 @@ const map crmThreshTypeResMap = { "mpls_nexthop_threshold_type", CrmResourceType::CRM_MPLS_NEXTHOP }, { "srv6_my_sid_entry_threshold_type", CrmResourceType::CRM_SRV6_MY_SID_ENTRY }, { "srv6_nexthop_threshold_type", CrmResourceType::CRM_SRV6_NEXTHOP }, + { "nexthop_group_map_threshold_type", CrmResourceType::CRM_NEXTHOP_GROUP_MAP }, }; const map crmThreshLowResMap = @@ -117,6 +120,7 @@ const map crmThreshLowResMap = {"mpls_nexthop_low_threshold", CrmResourceType::CRM_MPLS_NEXTHOP }, {"srv6_my_sid_entry_low_threshold", CrmResourceType::CRM_SRV6_MY_SID_ENTRY }, {"srv6_nexthop_low_threshold", CrmResourceType::CRM_SRV6_NEXTHOP }, + {"nexthop_group_map_low_threshold", CrmResourceType::CRM_NEXTHOP_GROUP_MAP }, }; const map crmThreshHighResMap = @@ -141,6 +145,7 @@ const map crmThreshHighResMap = {"mpls_nexthop_high_threshold", CrmResourceType::CRM_MPLS_NEXTHOP }, {"srv6_my_sid_entry_high_threshold", CrmResourceType::CRM_SRV6_MY_SID_ENTRY }, {"srv6_nexthop_high_threshold", CrmResourceType::CRM_SRV6_NEXTHOP }, + {"nexthop_group_map_high_threshold", CrmResourceType::CRM_NEXTHOP_GROUP_MAP }, }; const map crmThreshTypeMap = @@ -172,6 +177,7 @@ const map crmAvailCntsTableMap = { "crm_stats_mpls_nexthop_available", CrmResourceType::CRM_MPLS_NEXTHOP }, { "crm_stats_srv6_my_sid_entry_available", CrmResourceType::CRM_SRV6_MY_SID_ENTRY }, { "crm_stats_srv6_nexthop_available", CrmResourceType::CRM_SRV6_NEXTHOP }, + { "crm_stats_nexthop_group_map_available", CrmResourceType::CRM_NEXTHOP_GROUP_MAP }, }; const map crmUsedCntsTableMap = @@ -196,6 +202,7 @@ const map crmUsedCntsTableMap = { "crm_stats_mpls_nexthop_used", CrmResourceType::CRM_MPLS_NEXTHOP }, { "crm_stats_srv6_my_sid_entry_used", CrmResourceType::CRM_SRV6_MY_SID_ENTRY }, { "crm_stats_srv6_nexthop_used", CrmResourceType::CRM_SRV6_NEXTHOP }, + { "crm_stats_nexthop_group_map_used", CrmResourceType::CRM_NEXTHOP_GROUP_MAP }, }; CrmOrch::CrmOrch(DBConnector *db, string tableName): @@ -573,6 +580,7 @@ void CrmOrch::getResAvailableCounters() } case CrmResourceType::CRM_MPLS_INSEG: + case CrmResourceType::CRM_NEXTHOP_GROUP_MAP: { sai_object_type_t objType = static_cast(crmResSaiAvailAttrMap.at(res.first)); uint64_t availCount = 0; diff --git a/orchagent/crmorch.h b/orchagent/crmorch.h index 0218476398..345caa2cf6 100644 --- a/orchagent/crmorch.h +++ b/orchagent/crmorch.h @@ -32,6 +32,7 @@ enum class CrmResourceType CRM_MPLS_NEXTHOP, CRM_SRV6_MY_SID_ENTRY, CRM_SRV6_NEXTHOP, + CRM_NEXTHOP_GROUP_MAP, }; enum class CrmThresholdType From 80f3ac5835cfa0111b856304208619cdcef7d62b Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Mon, 8 Nov 2021 11:48:05 +0000 Subject: [PATCH 20/29] Fixes --- orchagent/cbf/cbfnhgorch.cpp | 2 +- tests/test_nhg.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/orchagent/cbf/cbfnhgorch.cpp b/orchagent/cbf/cbfnhgorch.cpp index 4ccbe7cda6..403945c7a9 100644 --- a/orchagent/cbf/cbfnhgorch.cpp +++ b/orchagent/cbf/cbfnhgorch.cpp @@ -308,7 +308,7 @@ bool CbfNhg::sync() nhg_attr.value.u32 = static_cast(m_members.size()); nhg_attrs.push_back(move(nhg_attr)); - if (nhg_attr.value.u32 > gNhgMapOrch->getMaxFcVal) + if (nhg_attr.value.u32 > gNhgMapOrch->getMaxFcVal()) { /* If there are more members than FCs then this may be an error, as some members won't be used. */ SWSS_LOG_WARN("More CBF NHG members configured than supported Forwarding Classes"); diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 8e4073eb2e..b5bdc1c48a 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -1813,7 +1813,6 @@ def update_deleting_cbf_nhg_test(): # Update the CBF group nhgm_ids = self.get_nhgm_ids('cbfgroup1') nhg_id = self.get_nhg_id('cbfgroup1') - old_map = self.asic_db.get_entry(self.ASIC_NHG_STR, nhg_id)['SAI_NEXT_HOP_GROUP_ATTR_SELECTION_MAP'] fvs = swsscommon.FieldValuePairs([('members', 'group2,group3'), ('selection_map', 'cbfnhgmap1')]) self.cbf_nhg_ps.set('cbfgroup1', fvs) self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) From 1215907df53602d213d6fb938c68b1ee628bb263 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Mon, 8 Nov 2021 13:28:44 +0000 Subject: [PATCH 21/29] Fixes --- tests/test_nhg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index b5bdc1c48a..616bb22157 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -1812,7 +1812,6 @@ def update_deleting_cbf_nhg_test(): # Update the CBF group nhgm_ids = self.get_nhgm_ids('cbfgroup1') - nhg_id = self.get_nhg_id('cbfgroup1') fvs = swsscommon.FieldValuePairs([('members', 'group2,group3'), ('selection_map', 'cbfnhgmap1')]) self.cbf_nhg_ps.set('cbfgroup1', fvs) self.asic_db.wait_for_deleted_keys(self.ASIC_NHGM_STR, nhgm_ids) From f0a4b950f5574cc7fd2785b374fd2e0ee18781b1 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 9 Nov 2021 10:31:23 +0000 Subject: [PATCH 22/29] QoS fix and UT --- orchagent/qosorch.cpp | 1 + tests/test_qos_map.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/orchagent/qosorch.cpp b/orchagent/qosorch.cpp index ae1e273634..f938d7fbf5 100644 --- a/orchagent/qosorch.cpp +++ b/orchagent/qosorch.cpp @@ -116,6 +116,7 @@ task_process_status QosMapHandler::processWorkItem(Consumer& consumer) vector attributes; if (!convertFieldValuesToAttributes(tuple, attributes)) { + freeAttribResources(attributes); return task_process_status::task_invalid_entry; } if (SAI_NULL_OBJECT_ID != sai_object) diff --git a/tests/test_qos_map.py b/tests/test_qos_map.py index c8eb3951b6..21a25742c9 100644 --- a/tests/test_qos_map.py +++ b/tests/test_qos_map.py @@ -152,6 +152,19 @@ def test_dscp_to_fc(self, dvs): fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) assert(fvs.get("SAI_QOS_MAP_ATTR_TYPE") == "SAI_QOS_MAP_TYPE_DSCP_TO_FORWARDING_CLASS") + # Modify the map + dscp_map = [(str(i), '0') for i in range(0, 64)] + self.dscp_ps.set("AZURE", swsscommon.FieldValuePairs(dscp_map)) + time.sleep(1) + + # Assert the expected values + fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) + sai_dscp_map = json.loads(fvs.get("SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST")) + + for dscp2fc in sai_dscp_map['list']: + fc = str(dscp2fc['value']['fc']) + assert fc == '0' + # Delete the map self.dscp_ps._del("AZURE") self.asic_db.wait_for_deleted_entry(self.ASIC_QOS_MAP_STR, dscp_map_id) @@ -193,6 +206,19 @@ def test_exp_to_fc(self, dvs): fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, exp_map_id) assert(fvs.get("SAI_QOS_MAP_ATTR_TYPE") == "SAI_QOS_MAP_TYPE_MPLS_EXP_TO_FORWARDING_CLASS") + # Modify the map + exp_map = [(str(i), '0') for i in range(0, 8)] + self.exp_ps.set("AZURE", swsscommon.FieldValuePairs(exp_map)) + time.sleep(1) + + # Assert the expected values + fvs = self.asic_db.get_entry(self.ASIC_QOS_MAP_STR, exp_map_id) + sai_exp_map = json.loads(fvs.get("SAI_QOS_MAP_ATTR_MAP_TO_VALUE_LIST")) + + for exp2fc in sai_exp_map['list']: + fc = str(exp2fc['value']['fc']) + assert fc == '0' + # Delete the map self.exp_ps._del("AZURE") self.asic_db.wait_for_deleted_entry(self.ASIC_QOS_MAP_STR, exp_map_id) From ea24f1f12210ee84041602634c3eb4e178dc4279 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 9 Nov 2021 15:08:53 +0000 Subject: [PATCH 23/29] UT --- tests/test_nhg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 616bb22157..30a7f441a1 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -89,6 +89,7 @@ def get_nhg_map_id(self, nhg_map_index): # Add a CBF NHG pointing to the given map fvs = swsscommon.FieldValuePairs([('members', 'testnhg'), ('selection_map', nhg_map_index)]) cbf_nhg_ps.set('testcbfnhg', fvs) + time.sleep(1) # If the CBF NHG can't be created, the provided NHG map index is invalid try: From cb7d98f3b923db15ab32ff0320013b6ae12743ab Mon Sep 17 00:00:00 2001 From: Alexandru Banu Date: Tue, 9 Nov 2021 14:51:10 +0200 Subject: [PATCH 24/29] Fix orchagent crash --- orchagent/mplsrouteorch.cpp | 10 ++++++++-- orchagent/nhgorch.cpp | 6 ++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/orchagent/mplsrouteorch.cpp b/orchagent/mplsrouteorch.cpp index beb0a33688..122bb6e8e1 100644 --- a/orchagent/mplsrouteorch.cpp +++ b/orchagent/mplsrouteorch.cpp @@ -520,8 +520,14 @@ bool RouteOrch::addLabelRoute(LabelRouteBulkContext& ctx, const NextHopGroupKey m_neighOrch->isNeighborResolved(nexthop)) { /* since IP neighbor NH exists, neighbor is resolved, add MPLS NH */ - m_neighOrch->addNextHop(nexthop); - next_hop_id = m_neighOrch->getNextHopId(nexthop); + if (m_neighOrch->addNextHop(nexthop)) + { + next_hop_id = m_neighOrch->getNextHopId(nexthop); + } + else + { + return false; + } } /* IP neighbor is not yet resolved */ else diff --git a/orchagent/nhgorch.cpp b/orchagent/nhgorch.cpp index b29010f25f..32ddb27eb5 100644 --- a/orchagent/nhgorch.cpp +++ b/orchagent/nhgorch.cpp @@ -380,8 +380,10 @@ sai_object_id_t NextHopGroupMember::getNhId() const */ else if (isLabeled() && gNeighOrch->isNeighborResolved(m_key)) { - gNeighOrch->addNextHop(m_key); - nh_id = gNeighOrch->getNextHopId(m_key); + if (gNeighOrch->addNextHop(m_key)) + { + nh_id = gNeighOrch->getNextHopId(m_key); + } } else { From 97f9209eb168f071dd7f50678d295f4d4f7e0148 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Tue, 9 Nov 2021 16:26:55 +0000 Subject: [PATCH 25/29] QoS fix --- orchagent/qosorch.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/orchagent/qosorch.cpp b/orchagent/qosorch.cpp index f938d7fbf5..59cb801bb4 100644 --- a/orchagent/qosorch.cpp +++ b/orchagent/qosorch.cpp @@ -116,7 +116,6 @@ task_process_status QosMapHandler::processWorkItem(Consumer& consumer) vector attributes; if (!convertFieldValuesToAttributes(tuple, attributes)) { - freeAttribResources(attributes); return task_process_status::task_invalid_entry; } if (SAI_NULL_OBJECT_ID != sai_object) @@ -825,11 +824,13 @@ bool DscpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple & if (value < 0) { SWSS_LOG_ERROR("DSCP value %d is negative", value); + delete[] list_attr.value.qosmap.list; return false; } else if (value > DSCP_MAX_VAL) { SWSS_LOG_ERROR("DSCP value %d is greater than max value %d", value, DSCP_MAX_VAL); + delete[] list_attr.value.qosmap.list; return false; } list_attr.value.qosmap.list[ind].key.dscp = static_cast(value); @@ -838,6 +839,7 @@ bool DscpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple & if ((value < 0) || (value > max_fc_val)) { SWSS_LOG_ERROR("FC value %d is either negative, or bigger than max value %d", value, max_fc_val); + delete[] list_attr.value.qosmap.list; return false; } list_attr.value.qosmap.list[ind].value.fc = static_cast(value); @@ -849,6 +851,7 @@ bool DscpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple & catch(const invalid_argument& e) { SWSS_LOG_ERROR("Got exception during conversion: %s", e.what()); + delete[] list_attr.value.qosmap.list; return false; } } @@ -914,11 +917,13 @@ bool ExpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &t if (value < 0) { SWSS_LOG_ERROR("EXP value %d is negative", value); + delete[] list_attr.value.qosmap.list; return false; } else if (value > EXP_MAX_VAL) { SWSS_LOG_ERROR("EXP value %d is greater than max value %d", value, EXP_MAX_VAL); + delete[] list_attr.value.qosmap.list; return false; } list_attr.value.qosmap.list[ind].key.mpls_exp = static_cast(value); @@ -927,6 +932,7 @@ bool ExpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &t if ((value < 0) || (value > max_fc_val)) { SWSS_LOG_ERROR("FC value %d is either negative, or bigger than max value %hu", value, max_fc_val); + delete[] list_attr.value.qosmap.list; return false; } list_attr.value.qosmap.list[ind].value.fc = static_cast(value); @@ -938,6 +944,7 @@ bool ExpToFcMapHandler::convertFieldValuesToAttributes(KeyOpFieldsValuesTuple &t catch(const invalid_argument& e) { SWSS_LOG_ERROR("Got exception during conversion: %s", e.what()); + delete[] list_attr.value.qosmap.list; return false; } } From 8fe699939f304b31ebbfdc44d957222b1bd31f86 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 10 Nov 2021 09:50:48 +0000 Subject: [PATCH 26/29] Fix UT --- tests/test_nhg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 30a7f441a1..781435dbe0 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -2061,6 +2061,7 @@ def nhg_map_exhaust_test(): self.fc_to_nhg_ps.set('cbfnhgmap512', fvs) time.sleep(1) assert(not self.nhg_map_exists('cbfnhgmap512')) + time.sleep(1) # Delete an existing NHG map. The pending NHG map should now be created del_nhg_map_index = nhg_maps.pop(0) From 922b5358a8a185a137bb5c03712ac0ba999e365a Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 10 Nov 2021 18:02:46 +0000 Subject: [PATCH 27/29] Fix merge issue --- orchagent/Makefile.am | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index 9f03aa6a77..ed30d16434 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -4,7 +4,7 @@ INCLUDES = -I $(top_srcdir)/lib \ -I flex_counter \ -I debug_counter \ -I pbh \ - -I cbf + -I nhg CFLAGS_SAI = -I /usr/include/sai @@ -41,10 +41,11 @@ orchagent_SOURCES = \ orchdaemon.cpp \ orch.cpp \ notifications.cpp \ - nhgbase.cpp \ nhgorch.cpp \ - cbf/cbfnhgorch.cpp \ - cbf/nhgmaporch.cpp \ + nhg/nhgbase.cpp \ + nhg/nhghandler.cpp \ + nhg/cbfnhghandler.cpp \ + nhgmaporch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ neighorch.cpp \ From 9cff8112312bc4fa5604b2829853503c0843c365 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Wed, 10 Nov 2021 18:36:51 +0000 Subject: [PATCH 28/29] Fix merge issue --- orchagent/Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index ed30d16434..f0d3fb8692 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -42,9 +42,9 @@ orchagent_SOURCES = \ orch.cpp \ notifications.cpp \ nhgorch.cpp \ - nhg/nhgbase.cpp \ - nhg/nhghandler.cpp \ - nhg/cbfnhghandler.cpp \ + cbf/nhgbase.cpp \ + cbf/nhghandler.cpp \ + cbf/cbfnhghandler.cpp \ nhgmaporch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ From 9ab252251f91729ab75559697d4cf3ef34267992 Mon Sep 17 00:00:00 2001 From: Tom cappleman Date: Thu, 11 Nov 2021 10:01:15 +0000 Subject: [PATCH 29/29] Fix and more UT --- orchagent/Makefile.am | 7 +++---- tests/test_nhg.py | 44 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/orchagent/Makefile.am b/orchagent/Makefile.am index f0d3fb8692..364fff4853 100644 --- a/orchagent/Makefile.am +++ b/orchagent/Makefile.am @@ -42,10 +42,9 @@ orchagent_SOURCES = \ orch.cpp \ notifications.cpp \ nhgorch.cpp \ - cbf/nhgbase.cpp \ - cbf/nhghandler.cpp \ - cbf/cbfnhghandler.cpp \ - nhgmaporch.cpp \ + nhgbase.cpp \ + cbf/cbfnhgorch.cpp \ + cbf/nhgmaporch.cpp \ routeorch.cpp \ mplsrouteorch.cpp \ neighorch.cpp \ diff --git a/tests/test_nhg.py b/tests/test_nhg.py index 781435dbe0..df071b4d17 100644 --- a/tests/test_nhg.py +++ b/tests/test_nhg.py @@ -1769,17 +1769,17 @@ def update_cbf_nhg_map_test(): def delete_referenced_cbf_nhg_test(): # Create a route pointing to the CBF NHG fvs = swsscommon.FieldValuePairs([('nexthop_group', 'cbfgroup1')]) - self.rt_ps.set('2.2.2.0/24', fvs) - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count + 1) + self.lr_ps.set('10', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count + 1) # Try deleting the CBF NHG - should not work self.cbf_nhg_ps._del('cbfgroup1') time.sleep(1) assert(self.nhg_exists('cbfgroup1')) - # Delete the route - the CBF NHG should also get deleted - self.rt_ps._del('2.2.2.0/24') - self.asic_db.wait_for_n_keys(self.ASIC_RT_STR, self.asic_rts_count) + # Delete the label route - the CBF NHG should also get deleted + self.lr_ps._del('10') + self.asic_db.wait_for_n_keys(self.ASIC_INSEG_STR, self.asic_insgs_count) self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) self.asic_db.wait_for_n_keys(self.ASIC_NHGM_STR, self.asic_nhgms_count + 3) @@ -1920,8 +1920,21 @@ def test_nhg_map(self, dvs, testlog): # Test scenario: # - create a NHG map and assert the expected details + # - update the map # - delete the map def mainline_nhg_map_test(): + # Create two non-CBF NHGs + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3,10.0.0.5'), + ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + self.nhg_ps.set("group1", fvs) + + fvs = swsscommon.FieldValuePairs([('nexthop', '10.0.0.1,10.0.0.3'), + ('ifname', 'Ethernet0,Ethernet4')]) + self.nhg_ps.set('group2', fvs) + + # Wait for the groups to appear in ASIC DB + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 2) + # Create an FC to NH index map fvs = swsscommon.FieldValuePairs([('0', '0')]) self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) @@ -1936,6 +1949,27 @@ def mainline_nhg_map_test(): {'SAI_NEXT_HOP_GROUP_MAP_ATTR_TYPE': 'SAI_NEXT_HOP_GROUP_MAP_TYPE_FORWARDING_CLASS_TO_INDEX', 'SAI_NEXT_HOP_GROUP_MAP_ATTR_MAP_TO_VALUE_LIST': '{\"count\":1,\"list\":[{\"key\":0,\"value\":0}]}'}) + # Create a CBF next hop group + fvs = swsscommon.FieldValuePairs([('members', 'group1,group2'), + ('selection_map', 'cbfnhgmap1')]) + self.cbf_nhg_ps.set('cbfgroup1', fvs) + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count + 3) + + # Update the map. + fvs = swsscommon.FieldValuePairs([('0', '0'), ('1', '1')]) + self.fc_to_nhg_ps.set('cbfnhgmap1', fvs) + + self.asic_db.wait_for_field_match(self.ASIC_NHG_MAP_STR, + nhg_map_id, + {'SAI_NEXT_HOP_GROUP_MAP_ATTR_TYPE': 'SAI_NEXT_HOP_GROUP_MAP_TYPE_FORWARDING_CLASS_TO_INDEX', + 'SAI_NEXT_HOP_GROUP_MAP_ATTR_MAP_TO_VALUE_LIST': '{\"count\":2,\"list\":[{\"key\":1,\"value\":1},{\"key\":0,\"value\":0}]}'}) + + # Delete the group + self.cbf_nhg_ps._del('cbfgroup1') + self.nhg_ps._del("group1") + self.nhg_ps._del("group2") + self.asic_db.wait_for_n_keys(self.ASIC_NHG_STR, self.asic_nhgs_count) + # Delete the map self.fc_to_nhg_ps._del('cbfnhgmap1') self.asic_db.wait_for_n_keys(self.ASIC_NHG_MAP_STR, self.asic_nhg_maps_count)