[pfcwd]: Rx/Tx/Drop counters (sonic-net#363)

* [pfcwd]: Rx/Tx/Drop counters

Keep track of Tx/Rx/Drop packets during watchdog action.
Record both accumulative values and counters from last event.

Signed-off-by: marian-pritsak <marianp@mellanox.com>

* Address comments

Signed-off-by: marian-pritsak <marianp@mellanox.com>

* Address comments

Signed-off-by: marian-pritsak <marianp@mellanox.com>

orchagent/pfcactionhandler.cpp

@@ -12,8 +12,15 @@
 #define PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED "PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED"
 #define PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED "PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED"
 
-#define SAI_QUEUE_STAT_PACKETS_STR         "SAI_QUEUE_STAT_PACKETS"
-#define SAI_QUEUE_STAT_DROPPED_PACKETS_STR "SAI_QUEUE_STAT_DROPPED_PACKETS"
+#define PFC_WD_QUEUE_STATS_TX_PACKETS "PFC_WD_QUEUE_STATS_TX_PACKETS"
+#define PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS "PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS"
+#define PFC_WD_QUEUE_STATS_RX_PACKETS "PFC_WD_QUEUE_STATS_RX_PACKETS"
+#define PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS "PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS"
+
+#define PFC_WD_QUEUE_STATS_TX_PACKETS_LAST "PFC_WD_QUEUE_STATS_TX_PACKETS_LAST"
+#define PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST "PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST"
+#define PFC_WD_QUEUE_STATS_RX_PACKETS_LAST "PFC_WD_QUEUE_STATS_RX_PACKETS_LAST"
+#define PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST "PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST"
 
 extern sai_object_id_t gSwitchId;
 extern PortsOrch *gPortsOrch;
@@ -35,30 +42,60 @@ PfcWdActionHandler::PfcWdActionHandler(sai_object_id_t port, sai_object_id_t que
             "PFC Watchdog detected PFC storm on queue 0x%lx port 0x%lx",
             m_queue,
             m_port);
-
-    m_stats = getQueueStats(m_countersTable, sai_serialize_object_id(m_queue));
-    m_stats.detectCount++;
-    m_stats.operational = false;
-
-    updateWdCounters(sai_serialize_object_id(m_queue), m_stats);
 }
 
 PfcWdActionHandler::~PfcWdActionHandler(void)
 {
     SWSS_LOG_ENTER();
 
-    auto finalStats = getQueueStats(m_countersTable, sai_serialize_object_id(m_queue));
-
     SWSS_LOG_NOTICE(
-            "Queue 0x%lx port 0x%lx restored from PFC storm. Tx packets: %lu. Dropped packets: %lu",
+            "Queue 0x%lx port 0x%lx restored from PFC storm.",
             m_queue,
-            m_port,
-            finalStats.txPkt - m_stats.txPkt,
-            finalStats.txDropPkt - m_stats.txDropPkt);
+            m_port);
+}
+
+void PfcWdActionHandler::initCounters(void)
+{
+    SWSS_LOG_ENTER();
+
+    if (!getHwCounters(m_hwStats))
+    {
+        return;
+    }
+
+    auto wdQueueStats = getQueueStats(m_countersTable, sai_serialize_object_id(m_queue));
+    wdQueueStats.detectCount++;
+    wdQueueStats.operational = false;
+
+    updateWdCounters(sai_serialize_object_id(m_queue), wdQueueStats);
+}
+
+void PfcWdActionHandler::commitCounters(void)
+{
+    SWSS_LOG_ENTER();
+
+    PfcWdHwStats hwStats;
+
+    if (!getHwCounters(hwStats))
+    {
+        return;
+    }
+
+    auto finalStats = getQueueStats(m_countersTable, sai_serialize_object_id(m_queue));
 
     finalStats.restoreCount++;
     finalStats.operational = true;
 
+    finalStats.txPktLast = hwStats.txPkt - m_hwStats.txPkt;
+    finalStats.txDropPktLast = hwStats.txDropPkt - m_hwStats.txDropPkt;
+    finalStats.rxPktLast = hwStats.rxPkt - m_hwStats.rxPkt;
+    finalStats.rxDropPktLast = hwStats.rxDropPkt - m_hwStats.rxDropPkt;
+
+    finalStats.txPkt += finalStats.txPktLast;
+    finalStats.txDropPkt += finalStats.txDropPktLast;
+    finalStats.rxPkt += finalStats.rxPktLast;
+    finalStats.rxDropPkt += finalStats.rxDropPktLast;
+
     updateWdCounters(sai_serialize_object_id(m_queue), finalStats);
 }
 
@@ -67,6 +104,8 @@ PfcWdActionHandler::PfcWdQueueStats PfcWdActionHandler::getQueueStats(shared_ptr
     SWSS_LOG_ENTER();
 
     PfcWdQueueStats stats;
+    memset(&stats, 0, sizeof(PfcWdQueueStats));
+    stats.operational = true;
     vector<FieldValueTuple> fieldValues;
 
     if (!countersTable->get(queueIdStr, fieldValues))
@@ -91,14 +130,22 @@ PfcWdActionHandler::PfcWdQueueStats PfcWdActionHandler::getQueueStats(shared_ptr
         {
             stats.operational = value == PFC_WD_QUEUE_STATUS_OPERATIONAL ? true : false;
         }
-        else if (field == SAI_QUEUE_STAT_PACKETS_STR)
+        else if (field == PFC_WD_QUEUE_STATS_TX_PACKETS)
         {
             stats.txPkt = stoul(value);
         }
-        else if (field == SAI_QUEUE_STAT_DROPPED_PACKETS_STR)
+        else if (field == PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS)
         {
             stats.txDropPkt = stoul(value);
         }
+        else if (field == PFC_WD_QUEUE_STATS_RX_PACKETS)
+        {
+            stats.rxPkt = stoul(value);
+        }
+        else if (field == PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS)
+        {
+            stats.rxDropPkt = stoul(value);
+        }
     }
 
     return move(stats);
@@ -127,6 +174,17 @@ void PfcWdActionHandler::updateWdCounters(const string& queueIdStr, const PfcWdQ
 
     resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_DEADLOCK_DETECTED, to_string(stats.detectCount));
     resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_DEADLOCK_RESTORED, to_string(stats.restoreCount));
+
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_TX_PACKETS, to_string(stats.txPkt));
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS, to_string(stats.txDropPkt));
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_RX_PACKETS, to_string(stats.rxPkt));
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS, to_string(stats.rxDropPkt));
+
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_TX_PACKETS_LAST, to_string(stats.txPktLast));
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_TX_DROPPED_PACKETS_LAST, to_string(stats.txDropPktLast));
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_RX_PACKETS_LAST, to_string(stats.rxPktLast));
+    resultFvValues.emplace_back(PFC_WD_QUEUE_STATS_RX_DROPPED_PACKETS_LAST, to_string(stats.rxDropPktLast));
+
     resultFvValues.emplace_back(PFC_WD_QUEUE_STATUS, stats.operational ?
                                                      PFC_WD_QUEUE_STATUS_OPERATIONAL :
                                                      PFC_WD_QUEUE_STATUS_STORMED);
@@ -292,6 +350,69 @@ PfcWdLossyHandler::~PfcWdLossyHandler(void)
     }
 }
 
+bool PfcWdLossyHandler::getHwCounters(PfcWdHwStats& counters)
+{
+    SWSS_LOG_ENTER();
+
+    static const vector<sai_queue_stat_t> queueStatIds =
+    {
+        SAI_QUEUE_STAT_PACKETS,
+        SAI_QUEUE_STAT_DROPPED_PACKETS,
+    };
+
+    static const vector<sai_ingress_priority_group_stat_t> pgStatIds =
+    {
+        SAI_INGRESS_PRIORITY_GROUP_STAT_PACKETS,
+        SAI_INGRESS_PRIORITY_GROUP_STAT_DROPPED_PACKETS,
+    };
+
+    vector<uint64_t> queueStats;
+    queueStats.resize(queueStatIds.size());
+
+    sai_status_t status = sai_queue_api->get_queue_stats(
+            getQueue(),
+            static_cast<uint32_t>(queueStatIds.size()),
+            queueStatIds.data(),
+            queueStats.data());
+
+    if (status != SAI_STATUS_SUCCESS)
+    {
+        SWSS_LOG_ERROR("Failed to fetch queue 0x%lx stats: %d", getQueue(), status);
+        return false;
+    }
+
+    // PG counters not yet supported in Mellanox platform
+    Port portInstance;
+    if (!gPortsOrch->getPort(getPort(), portInstance))
+    {
+        SWSS_LOG_ERROR("Cannot get port by ID 0x%lx", getPort());
+        return false;
+    }
+
+    sai_object_id_t pg = portInstance.m_priority_group_ids[getQueueId()];
+    vector<uint64_t> pgStats;
+    pgStats.resize(pgStatIds.size());
+
+    status = sai_buffer_api->get_ingress_priority_group_stats(
+            pg,
+            static_cast<uint32_t>(pgStatIds.size()),
+            pgStatIds.data(),
+            pgStats.data());
+
+    if (status != SAI_STATUS_SUCCESS)
+    {
+        SWSS_LOG_ERROR("Failed to fetch pg 0x%lx stats: %d", pg, status);
+        return false;
+    }
+
+    counters.txPkt = queueStats[0];
+    counters.txDropPkt = queueStats[1];
+    counters.rxPkt = pgStats[0];
+    counters.rxDropPkt = pgStats[1];
+
+    return true;
+}
+
 PfcWdZeroBufferHandler::PfcWdZeroBufferHandler(sai_object_id_t port,
         sai_object_id_t queue, uint8_t queueId, shared_ptr<Table> countersTable):
     PfcWdLossyHandler(port, queue, queueId, countersTable)

orchagent/pfcactionhandler.h

@@ -13,6 +13,14 @@ extern "C" {
 using namespace std;
 using namespace swss;
 
+struct PfcWdHwStats
+{
+    uint64_t txPkt;
+    uint64_t txDropPkt;
+    uint64_t rxPkt;
+    uint64_t rxDropPkt;
+};
+
 // PFC queue interface class
 // It resembles RAII behavior - pause storm is mitigated (queue is locked) on creation,
 // and is restored (queue released) on removal
@@ -39,15 +47,30 @@ class PfcWdActionHandler
         }
 
         static void initWdCounters(shared_ptr<Table> countersTable, const string &queueIdStr);
+        void initCounters(void);
+        void commitCounters(void);
+
+        virtual bool getHwCounters(PfcWdHwStats& counters)
+        {
+            memset(&counters, 0, sizeof(PfcWdHwStats));
+
+            return true;
+        };
 
     private:
         struct PfcWdQueueStats
         {
-            uint64_t detectCount  = 0;
-            uint64_t restoreCount = 0;
-            uint64_t txPkt        = 0;
-            uint64_t txDropPkt    = 0;
-            bool     operational  = true;
+            uint64_t detectCount;
+            uint64_t restoreCount;
+            uint64_t txPkt;
+            uint64_t txDropPkt;
+            uint64_t rxPkt;
+            uint64_t rxDropPkt;
+            uint64_t txPktLast;
+            uint64_t txDropPktLast;
+            uint64_t rxPktLast;
+            uint64_t rxDropPktLast;
+            bool     operational;
         };
 
         static PfcWdQueueStats getQueueStats(shared_ptr<Table> countersTable, const string &queueIdStr);
@@ -57,7 +80,7 @@ class PfcWdActionHandler
         sai_object_id_t m_queue = SAI_NULL_OBJECT_ID;
         uint8_t m_queueId = 0;
         shared_ptr<Table> m_countersTable = nullptr;
-        PfcWdQueueStats m_stats;
+        PfcWdHwStats m_hwStats;
 };
 
 class PfcWdAclHandler: public PfcWdActionHandler
@@ -87,6 +110,7 @@ class PfcWdLossyHandler: public PfcWdActionHandler
         PfcWdLossyHandler(sai_object_id_t port, sai_object_id_t queue,
                 uint8_t queueId, shared_ptr<Table> countersTable);
         virtual ~PfcWdLossyHandler(void);
+        virtual bool getHwCounters(PfcWdHwStats& counters);
 };
 
 // PFC queue that implements drop action by draining queue with buffer of zero size

orchagent/pfcwdorch.cpp

@@ -506,9 +506,11 @@ void PfcWdSwOrch<DropHandler, ForwardHandler>::doTask(swss::NotificationConsumer
         {
             throw runtime_error("Unknown PFC WD action");
         }
+        entry->second.handler->initCounters();
     }
     else if (event == "restore")
     {
+        entry->second.handler->commitCounters();
         entry->second.handler = nullptr;
     }
     else