HDDS-9988. SCM UI shows storage usage percentage

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

@@ -69,7 +69,9 @@
 
 import javax.management.ObjectName;
 import java.io.IOException;
+import java.math.RoundingMode;
 import java.net.InetAddress;
+import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -138,9 +140,11 @@ public class SCMNodeManager implements NodeManager {
    * consistent view of the node state.
    */
   private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final String opeState = "OPSTATE";
-  private final String comState = "COMSTATE";
-  private final String lastHeartbeat = "LASTHEARTBEAT";
+  private static final String OPESTATE = "OPSTATE";
+  private static final String COMSTATE = "COMSTATE";
+  private static final String LASTHEARTBEAT = "LASTHEARTBEAT";
+  private static final String USEDSPACEPERCENT = "USEDSPACEPERCENT";
+  private static final String TOTALCAPACITY = "CAPACITY";
   /**
    * Constructs SCM machine Manager.
    */
@@ -1103,21 +1107,107 @@ public Map<String, Map<String, String>> getNodeStatusInfo() {
         heartbeatTimeDiff = getLastHeartbeatTimeDiff(dni.getLastHeartbeatTime());
       }
       Map<String, String> map = new HashMap<>();
-      map.put(opeState, opstate);
-      map.put(comState, healthState);
-      map.put(lastHeartbeat, heartbeatTimeDiff);
+      map.put(OPESTATE, opstate);
+      map.put(COMSTATE, healthState);
+      map.put(LASTHEARTBEAT, heartbeatTimeDiff);
       if (httpPort != null) {
         map.put(httpPort.getName().toString(), httpPort.getValue().toString());
       }
       if (httpsPort != null) {
         map.put(httpsPort.getName().toString(),
                   httpsPort.getValue().toString());
       }
+      String capacity = calculateStorageCapacity(dni.getStorageReports());
+      map.put(TOTALCAPACITY, capacity);
+      String[] storagePercentage = calculateStoragePercentage(
+          dni.getStorageReports());
+      String scmUsedPerc = storagePercentage[0];
+      String nonScmUsedPerc = storagePercentage[1];
+      map.put(USEDSPACEPERCENT,
+          "Ozone: " + scmUsedPerc + "%, other: " + nonScmUsedPerc + "%");
       nodes.put(hostName, map);
     }
     return nodes;
   }
 
+  /**
+   * Calculate the storage capacity of the DataNode node.
+   * @param storageReports Calculate the storage capacity corresponding
+   *                       to the storage collection.
+   * @return
+   */
+  public static String calculateStorageCapacity(
+      List<StorageReportProto> storageReports) {
+    long capacityByte = 0;
+    if (storageReports != null && !storageReports.isEmpty()) {
+      for (StorageReportProto storageReport : storageReports) {
+        capacityByte += storageReport.getCapacity();
+      }
+    }
+
+    double ua = capacityByte;
+    StringBuilder unit = new StringBuilder("B");
+    if (ua > 1024) {
+      ua = ua / 1024;
+      unit.replace(0, 1, "KB");
+    }
+    if (ua > 1024) {
+      ua = ua / 1024;
+      unit.replace(0, 2, "MB");
+    }
+    if (ua > 1024) {
+      ua = ua / 1024;
+      unit.replace(0, 2, "GB");
+    }
+    if (ua > 1024) {
+      ua = ua / 1024;
+      unit.replace(0, 2, "TB");
+    }
+
+    DecimalFormat decimalFormat = new DecimalFormat("#0.0");
+    decimalFormat.setRoundingMode(RoundingMode.HALF_UP);
+    String capacity = decimalFormat.format(ua);
+    return capacity + unit.toString();
+  }
+
+  /**
+   * Calculate the storage usage percentage of a DataNode node.
+   * @param storageReports Calculate the storage percentage corresponding
+   *                       to the storage collection.
+   * @return
+   */
+  public static String[] calculateStoragePercentage(
+      List<StorageReportProto> storageReports) {
+    String[] storagePercentage = new String[2];
+    String usedPercentage = "N/A";
+    String nonUsedPercentage = "N/A";
+    if (storageReports != null && !storageReports.isEmpty()) {
+      long capacity = 0;
+      long scmUsed = 0;
+      long remaining = 0;
+      for (StorageReportProto storageReport : storageReports) {
+        capacity += storageReport.getCapacity();
+        scmUsed += storageReport.getScmUsed();
+        remaining += storageReport.getRemaining();
+      }
+      long scmNonUsed = capacity - scmUsed - remaining;
+
+      DecimalFormat decimalFormat = new DecimalFormat("#0.00");
+      decimalFormat.setRoundingMode(RoundingMode.HALF_UP);
+
+      double usedPerc = ((double) scmUsed / capacity) * 100;
+      usedPerc = usedPerc > 100.0 ? 100.0 : usedPerc;
+      double nonUsedPerc = ((double) scmNonUsed / capacity) * 100;
+      nonUsedPerc = nonUsedPerc > 100.0 ? 100.0 : nonUsedPerc;
+      usedPercentage = decimalFormat.format(usedPerc);
+      nonUsedPercentage = decimalFormat.format(nonUsedPerc);
+    }
+
+    storagePercentage[0] = usedPercentage;
+    storagePercentage[1] = nonUsedPercentage;
+    return storagePercentage;
+  }
+
   /**
    * Based on the current time and the last heartbeat, calculate the time difference
    * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc.

hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html

@@ -48,6 +48,10 @@ <h2>Node Status</h2>
         <tr>
             <th ng-click = "columnSort('hostname')" class="nodeStatusInfo"><span ng-class = "{'sorting' : (columnName != 'hostname'), 'sortasc' : (columnName == 'hostname' && !reverse),
                                         'sortdesc':(columnName == 'hostname' && !reverse)}">HostName</span></th>
+            <th ng-click = "columnSort('usedspacepercent')" class="nodeStatusInfo" ><span ng-class="{'sorting' : (columnName != 'usedspacepercent'), 'sortasc' : (columnName == 'usedspacepercent' && !reverse),
+                                        'sortdesc':(columnName == 'usedspacepercent' && !reverse)}">Used Space Percent</span></th>
+            <th ng-click = "columnSort('capacity')" class="nodeStatusInfo" ><span ng-class="{'sorting' : (columnName != 'capacity'), 'sortasc' : (columnName == 'capacity' && !reverse),
+                                        'sortdesc':(columnName == 'capacity' && !reverse)}">Capacity</span></th>
             <th ng-click = "columnSort('opstate')" class="nodeStatusInfo" ><span ng-class="{'sorting' : (columnName != 'opstate'), 'sortasc' : (columnName == 'opstate' && !reverse),
                                         'sortdesc':(columnName == 'opstate' && !reverse)}">Operational State</span></th>
             <th ng-click = "columnSort('comstate')" class="nodeStatusInfo">  <span ng-class="{'sorting' : (columnName != 'comstate'), 'sortasc' : (columnName == 'comstate' && !reverse),
@@ -66,6 +70,8 @@ <h2>Node Status</h2>
                     {{typestat.hostname}}
                 </span>
             </td>
+            <td>{{typestat.usedspacepercent}}</td>
+            <td>{{typestat.capacity}}</td>
             <td>{{typestat.opstate}}</td>
             <td>{{typestat.comstate}}</td>
             <td>{{typestat.lastheartbeat}}</td>

hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js

@@ -68,6 +68,8 @@
                                 return {
                                     hostname: key,
                                     opstate: value && value.find((element) => element.key === "OPSTATE").value,
+                                    usedspacepercent: value && value.find((element) => element.key === "USEDSPACEPERCENT").value,
+                                    capacity: value && value.find((element) => element.key === "CAPACITY").value,
                                     comstate: value && value.find((element) => element.key === "COMSTATE").value,
                                     lastheartbeat: value && value.find((element) => element.key === "LASTHEARTBEAT").value,
                                     port: portSpec.port,

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java

@@ -86,6 +86,7 @@
 import java.util.Map;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -123,6 +124,8 @@
 import static org.mockito.Mockito.eq;
 
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.ArgumentCaptor;
 import org.slf4j.Logger;
@@ -1572,6 +1575,49 @@ public void testScmStatsFromNodeReport()
     }
   }
 
+  private List<StorageReportProto> generateStorageReportProto(
+      int volumeCount, UUID dnId, long capacity, long used, long remaining) {
+    List<StorageReportProto> reports = new ArrayList<>(volumeCount);
+    boolean failed = true;
+    for (int x = 0; x < volumeCount; x++) {
+      String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+      reports.add(HddsTestUtils
+          .createStorageReport(dnId, storagePath, capacity,
+              used, remaining, null, failed));
+      failed = !failed;
+    }
+    return reports;
+  }
+
+  private static Stream<Arguments> calculateStoragePercentageScenarios() {
+    return Stream.of(
+        Arguments.of(600, 65, 500, 1, "600.0B", "10.83", "5.83"),
+        Arguments.of(10000, 1000, 8800, 12, "117.2KB", "10.00", "2.00"),
+        Arguments.of(100000000, 1000, 899999, 12, "1.1GB", "0.00", "99.10"),
+        Arguments.of(10000, 1000, 0, 0, "0.0B", "N/A", "N/A"),
+        Arguments.of(0, 0, 0, 0, "0.0B", "N/A", "N/A"),
+        Arguments.of(1010, 547, 400, 5, "4.9KB", "54.16", "6.24")
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("calculateStoragePercentageScenarios")
+  public void testCalculateStoragePercentage(long perCapacity,
+      long used, long remaining, int volumeCount, String totalCapacity,
+          String scmUsedPerc, String nonScmUsedPerc) {
+    DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+    UUID dnId = dn.getUuid();
+    List<StorageReportProto> reports = volumeCount > 0 ?
+        generateStorageReportProto(volumeCount, dnId, perCapacity,
+            used, remaining) : null;
+    String capacityResult = SCMNodeManager.calculateStorageCapacity(reports);
+    assertEquals(totalCapacity, capacityResult);
+    String[] storagePercentage = SCMNodeManager.calculateStoragePercentage(
+        reports);
+    assertEquals(scmUsedPerc, storagePercentage[0]);
+    assertEquals(nonScmUsedPerc, storagePercentage[1]);
+  }
+
   /**
    * Test multiple nodes sending initial heartbeat with their node report
    * with multiple volumes.