diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index e9b7d220e1f2..cc5fb9aa776e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -69,7 +69,9 @@ import javax.management.ObjectName; import java.io.IOException; +import java.math.RoundingMode; import java.net.InetAddress; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -138,9 +140,11 @@ public class SCMNodeManager implements NodeManager { * consistent view of the node state. */ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private final String opeState = "OPSTATE"; - private final String comState = "COMSTATE"; - private final String lastHeartbeat = "LASTHEARTBEAT"; + private static final String OPESTATE = "OPSTATE"; + private static final String COMSTATE = "COMSTATE"; + private static final String LASTHEARTBEAT = "LASTHEARTBEAT"; + private static final String USEDSPACEPERCENT = "USEDSPACEPERCENT"; + private static final String TOTALCAPACITY = "CAPACITY"; /** * Constructs SCM machine Manager. */ @@ -1103,9 +1107,9 @@ public Map> getNodeStatusInfo() { heartbeatTimeDiff = getLastHeartbeatTimeDiff(dni.getLastHeartbeatTime()); } Map map = new HashMap<>(); - map.put(opeState, opstate); - map.put(comState, healthState); - map.put(lastHeartbeat, heartbeatTimeDiff); + map.put(OPESTATE, opstate); + map.put(COMSTATE, healthState); + map.put(LASTHEARTBEAT, heartbeatTimeDiff); if (httpPort != null) { map.put(httpPort.getName().toString(), httpPort.getValue().toString()); } @@ -1113,11 +1117,97 @@ public Map> getNodeStatusInfo() { map.put(httpsPort.getName().toString(), httpsPort.getValue().toString()); } + String capacity = calculateStorageCapacity(dni.getStorageReports()); + map.put(TOTALCAPACITY, capacity); + String[] storagePercentage = calculateStoragePercentage( + dni.getStorageReports()); + String scmUsedPerc = storagePercentage[0]; + String nonScmUsedPerc = storagePercentage[1]; + map.put(USEDSPACEPERCENT, + "Ozone: " + scmUsedPerc + "%, other: " + nonScmUsedPerc + "%"); nodes.put(hostName, map); } return nodes; } + /** + * Calculate the storage capacity of the DataNode node. + * @param storageReports Calculate the storage capacity corresponding + * to the storage collection. + * @return + */ + public static String calculateStorageCapacity( + List storageReports) { + long capacityByte = 0; + if (storageReports != null && !storageReports.isEmpty()) { + for (StorageReportProto storageReport : storageReports) { + capacityByte += storageReport.getCapacity(); + } + } + + double ua = capacityByte; + StringBuilder unit = new StringBuilder("B"); + if (ua > 1024) { + ua = ua / 1024; + unit.replace(0, 1, "KB"); + } + if (ua > 1024) { + ua = ua / 1024; + unit.replace(0, 2, "MB"); + } + if (ua > 1024) { + ua = ua / 1024; + unit.replace(0, 2, "GB"); + } + if (ua > 1024) { + ua = ua / 1024; + unit.replace(0, 2, "TB"); + } + + DecimalFormat decimalFormat = new DecimalFormat("#0.0"); + decimalFormat.setRoundingMode(RoundingMode.HALF_UP); + String capacity = decimalFormat.format(ua); + return capacity + unit.toString(); + } + + /** + * Calculate the storage usage percentage of a DataNode node. + * @param storageReports Calculate the storage percentage corresponding + * to the storage collection. + * @return + */ + public static String[] calculateStoragePercentage( + List storageReports) { + String[] storagePercentage = new String[2]; + String usedPercentage = "N/A"; + String nonUsedPercentage = "N/A"; + if (storageReports != null && !storageReports.isEmpty()) { + long capacity = 0; + long scmUsed = 0; + long remaining = 0; + for (StorageReportProto storageReport : storageReports) { + capacity += storageReport.getCapacity(); + scmUsed += storageReport.getScmUsed(); + remaining += storageReport.getRemaining(); + } + long scmNonUsed = capacity - scmUsed - remaining; + + DecimalFormat decimalFormat = new DecimalFormat("#0.00"); + decimalFormat.setRoundingMode(RoundingMode.HALF_UP); + + double usedPerc = ((double) scmUsed / capacity) * 100; + usedPerc = usedPerc > 100.0 ? 100.0 : usedPerc; + double nonUsedPerc = ((double) scmNonUsed / capacity) * 100; + nonUsedPerc = nonUsedPerc > 100.0 ? 100.0 : nonUsedPerc; + usedPercentage = decimalFormat.format(usedPerc); + nonUsedPercentage = decimalFormat.format(nonUsedPerc); + } + + storagePercentage[0] = usedPercentage; + storagePercentage[1] = nonUsedPercentage; + return storagePercentage; + } + /** * Based on the current time and the last heartbeat, calculate the time difference * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc. diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index 214a2ad7868a..fdd8de15b6a9 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -48,6 +48,10 @@

Node Status

HostName + Used Space Percent + Capacity Operational State element.key === "USEDSPACEPERCENT").value, + capacity: value && value.find((element) => element.key === "CAPACITY").value, comstate: value && value.find((element) => element.key === "COMSTATE").value, lastheartbeat: value && value.find((element) => element.key === "LASTHEARTBEAT").value, port: portSpec.port, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 85a70b646739..930774a54bf3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -86,6 +86,7 @@ import java.util.Map; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -123,6 +124,8 @@ import static org.mockito.Mockito.eq; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; @@ -1572,6 +1575,49 @@ public void testScmStatsFromNodeReport() } } + private List generateStorageReportProto( + int volumeCount, UUID dnId, long capacity, long used, long remaining) { + List reports = new ArrayList<>(volumeCount); + boolean failed = true; + for (int x = 0; x < volumeCount; x++) { + String storagePath = testDir.getAbsolutePath() + "/" + dnId; + reports.add(HddsTestUtils + .createStorageReport(dnId, storagePath, capacity, + used, remaining, null, failed)); + failed = !failed; + } + return reports; + } + + private static Stream calculateStoragePercentageScenarios() { + return Stream.of( + Arguments.of(600, 65, 500, 1, "600.0B", "10.83", "5.83"), + Arguments.of(10000, 1000, 8800, 12, "117.2KB", "10.00", "2.00"), + Arguments.of(100000000, 1000, 899999, 12, "1.1GB", "0.00", "99.10"), + Arguments.of(10000, 1000, 0, 0, "0.0B", "N/A", "N/A"), + Arguments.of(0, 0, 0, 0, "0.0B", "N/A", "N/A"), + Arguments.of(1010, 547, 400, 5, "4.9KB", "54.16", "6.24") + ); + } + + @ParameterizedTest + @MethodSource("calculateStoragePercentageScenarios") + public void testCalculateStoragePercentage(long perCapacity, + long used, long remaining, int volumeCount, String totalCapacity, + String scmUsedPerc, String nonScmUsedPerc) { + DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + UUID dnId = dn.getUuid(); + List reports = volumeCount > 0 ? + generateStorageReportProto(volumeCount, dnId, perCapacity, + used, remaining) : null; + String capacityResult = SCMNodeManager.calculateStorageCapacity(reports); + assertEquals(totalCapacity, capacityResult); + String[] storagePercentage = SCMNodeManager.calculateStoragePercentage( + reports); + assertEquals(scmUsedPerc, storagePercentage[0]); + assertEquals(nonScmUsedPerc, storagePercentage[1]); + } + /** * Test multiple nodes sending initial heartbeat with their node report * with multiple volumes.