HDDS-11206. Statistics storage usage indicators include min, max, median, stdev

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java

@@ -20,6 +20,7 @@
 
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 
+import java.util.HashMap;
 import java.util.Map;
 
 /**
@@ -48,4 +49,8 @@ public interface NodeManagerMXBean {
    */
   Map<String, Map<String, String>> getNodeStatusInfo();
 
+  default Map<String, String> getNodeStatistics() {
+    return new HashMap<>();
+  }
+
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

@@ -83,6 +83,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
+import java.util.Arrays;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -1217,6 +1218,53 @@ public static String[] calculateStoragePercentage(
     return storagePercentage;
   }
 
+  @Override
+  public Map<String, String> getNodeStatistics() {
+    Map<String, String> nodeStatics = new HashMap<>();
+    // Statistics node usaged
+    nodeUsageStatics(nodeStatics);
+    // todo: Statistics of other instances
+    return nodeStatics;
+  }
+
+  private void nodeUsageStatics(Map<String, String> nodeStatics) {
+    if (nodeStateManager.getAllNodes().size() < 1) {
+      return;
+    }
+    float[] usages = new float[nodeStateManager.getAllNodes().size()];
+    float totalOzoneUsed = 0;
+    int i = 0;
+    for (DatanodeInfo dni : nodeStateManager.getAllNodes()) {
+      String[] storagePercentage = calculateStoragePercentage(
+              dni.getStorageReports());
+      if (storagePercentage[0].equals("N/A")) {
+        usages[i++] = 0;
+      } else {
+        float storagePerc = Float.parseFloat(storagePercentage[0]);
+        usages[i++] = storagePerc;
+        totalOzoneUsed = totalOzoneUsed + storagePerc;
+      }
+    }
+
+    totalOzoneUsed /= nodeStateManager.getAllNodes().size();
+    Arrays.sort(usages);
+    float median = usages[usages.length / 2];
+    nodeStatics.put(UsageStatics.MEDIAN.getLabel(), String.valueOf(median));
+    float max = usages[usages.length - 1];
+    nodeStatics.put(UsageStatics.MAX.getLabel(), String.valueOf(max));
+    float min = usages[0];
+    nodeStatics.put(UsageStatics.MIN.getLabel(), String.valueOf(min));
+
+    float dev = 0;
+    for (i = 0; i < usages.length; i++) {
+      dev += (usages[i] - totalOzoneUsed) * (usages[i] - totalOzoneUsed);
+    }
+    dev = (float) Math.sqrt(dev / usages.length);
+    DecimalFormat decimalFormat = new DecimalFormat("#0.00");
+    decimalFormat.setRoundingMode(RoundingMode.HALF_UP);
+    nodeStatics.put(UsageStatics.STDEV.getLabel(), decimalFormat.format(dev));
+  }
+
   /**
    * Based on the current time and the last heartbeat, calculate the time difference
    * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc.
@@ -1284,6 +1332,20 @@ public String getLabel() {
     }
   }
 
+  private enum UsageStatics {
+    MIN("Min"),
+    MAX("Max"),
+    MEDIAN("Median"),
+    STDEV("Stdev");
+    private String label;
+    public String getLabel() {
+      return label;
+    }
+    UsageStatics(String label) {
+      this.label = label;
+    }
+  }
+
   /**
    * Returns the min of no healthy volumes reported out of the set
    * of datanodes constituting the pipeline.

hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html

@@ -28,6 +28,16 @@ <h2>SCM Information</h2>
     </tbody>
 </table>
 
+<h2>Statistical</h2>
+<table class="table table-bordered table-striped">
+    <tbody>
+    <tr>
+        <td>DataNodes usages% (Min/Median/Max/stdDev)</td>
+        <td>{{statistics.nodes.usages.min}}%/{{statistics.nodes.usages.median}}%/{{statistics.nodes.usages.max}}%/{{statistics.nodes.usages.stdev}}%</td>
+    </tr>
+    </tbody>
+</table>
+
 <h2>Node Status</h2>
 <div class="row">
     <div class="col-md-6 text-left">

hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js

@@ -32,6 +32,15 @@
             $scope.RecordsToDisplay = "10";
             $scope.currentPage = 1;
             $scope.lastIndex = 0;
+            $scope.statistics = {
+                nodes : {
+                    usages : {}
+                }
+            }
+            $scope.statistics.nodes.usages.min = "";
+            $scope.statistics.nodes.usages.max = "";
+            $scope.statistics.nodes.usages.median = "";
+            $scope.statistics.nodes.usages.stdev = "";
 
             function get_protocol(URLScheme, value, baseProto, fallbackProto) {
                 let protocol = "unknown"
@@ -81,6 +90,18 @@
                     $scope.totalItems = nodeStatusCopy.length;
                     $scope.lastIndex = Math.ceil(nodeStatusCopy.length / $scope.RecordsToDisplay);
                     $scope.nodeStatus = nodeStatusCopy.slice(0, $scope.RecordsToDisplay);
+
+                    ctrl.nodemanagermetrics.NodeStatistics.map(({ key, value }) => {
+                        if(key == "Min") {
+                            $scope.statistics.nodes.usages.min = value;
+                        } else if(key == "Max") {
+                            $scope.statistics.nodes.usages.max = value;
+                        } else if(key == "Median") {
+                            $scope.statistics.nodes.usages.median = value;
+                        } else if(key == "Stdev") {
+                            $scope.statistics.nodes.usages.stdev = value;
+                        }
+                    });
                 });
             /*if option is 'All' display all records else display specified record on page*/
             $scope.UpdateRecordsToShow = () => {