From 11a0564028922ef24003f74f48318571f6255b7a Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 7 Jan 2024 00:49:40 +0800 Subject: [PATCH 1/8] HDDS-9988. SCM UI shows storage usage percentage --- .../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +++++++ .../resources/webapps/scm/scm-overview.html | 3 + .../src/main/resources/webapps/scm/scm.js | 1 + .../hdds/scm/node/TestSCMNodeManager.java | 83 +++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index e9b7d220e1f..68cad9f7dc8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -69,7 +69,9 @@ import javax.management.ObjectName; import java.io.IOException; +import java.math.RoundingMode; import java.net.InetAddress; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -141,6 +143,7 @@ public class SCMNodeManager implements NodeManager { private final String opeState = "OPSTATE"; private final String comState = "COMSTATE"; private final String lastHeartbeat = "LASTHEARTBEAT"; + private final String usedPercent = "USEDPERCENT"; /** * Constructs SCM machine Manager. */ @@ -1113,11 +1116,37 @@ public Map> getNodeStatusInfo() { map.put(httpsPort.getName().toString(), httpsPort.getValue().toString()); } + double usedPec = calculateStoragePercentage(dni); + map.put(usedPercent, usedPec + "%"); nodes.put(hostName, map); } return nodes; } + /** + * Calculate the storage usage percentage of a DataNode node. + * @param dni DataNode node that needs to be calculated + * @return + */ + public double calculateStoragePercentage(DatanodeInfo dni) { + double usedPec = 0; + List storageReports = dni.getStorageReports(); + if (storageReports != null && !storageReports.isEmpty()) { + long capacity = 0; + long scmUsed = 0; + for (StorageReportProto storageReport : storageReports) { + capacity = capacity + storageReport.getCapacity(); + scmUsed = scmUsed + storageReport.getScmUsed(); + } + double percent = (double) scmUsed / capacity; + percent = percent > 1.0 ? 1.0 : percent; + DecimalFormat decimalFormat = new DecimalFormat("#0.0000"); + decimalFormat.setRoundingMode(RoundingMode.HALF_UP); + usedPec = Double.valueOf(decimalFormat.format(percent)) * 100; + } + return usedPec; + } + /** * Based on the current time and the last heartbeat, calculate the time difference * and get a string of the relative value. E.g. "2s ago", "1m 2s ago", etc. diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index 214a2ad7868..8e4d00ea9e3 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -48,6 +48,8 @@

Node Status

HostName + Used Percent Operational State element.key === "USEDPERCENT").value, comstate: value && value.find((element) => element.key === "COMSTATE").value, lastheartbeat: value && value.find((element) => element.key === "LASTHEARTBEAT").value, port: portSpec.port, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 85a70b64673..e1f9a83e554 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -1572,6 +1572,89 @@ public void testScmStatsFromNodeReport() } } + private List generateStorageReportProto( + int volumeCount, UUID dnId, long capacity, long used) { + List reports = new ArrayList<>(volumeCount); + boolean failed = true; + for (int x = 0; x < volumeCount; x++) { + long free = capacity - used; + String storagePath = testDir.getAbsolutePath() + "/" + dnId; + reports.add(HddsTestUtils + .createStorageReport(dnId, storagePath, capacity, + used, free, null, failed)); + failed = !failed; + } + return reports; + } + + @Test + public void testCalculateStoragePercentage() + throws IOException, AuthenticationException { + OzoneConfiguration conf = getConf(); + conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, + MILLISECONDS); + List dnList = new ArrayList<>(4); + try (SCMNodeManager nodeManager = createNodeManager(conf)) { + EventQueue eventQueue = (EventQueue) scm.getEventQueue(); + LayoutVersionManager versionManager = + nodeManager.getLayoutVersionManager(); + LayoutVersionProto layoutInfo = toLayoutVersionProto( + versionManager.getMetadataLayoutVersion(), + versionManager.getSoftwareLayoutVersion()); + // Generate the first DataNode + DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); + dnList.add(dn1); + UUID dnId1 = dn1.getUuid(); + int volumeCount = 10; + List reports1 = generateStorageReportProto( + volumeCount, dnId1, 1000, 600); + nodeManager.register(dn1, HddsTestUtils.createNodeReport(reports1, + emptyList()), null); + nodeManager.processHeartbeat(dn1, layoutInfo); + // Generate the 2nd DataNode + DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails(); + dnList.add(dn2); + UUID dnId2 = dn2.getUuid(); + List reports2 = generateStorageReportProto( + volumeCount, dnId2, 1000, 1000); + nodeManager.register(dn2, HddsTestUtils.createNodeReport(reports2, + emptyList()), null); + nodeManager.processHeartbeat(dn2, layoutInfo); + // Generate the 3rd DataNode + DatanodeDetails dn3 = MockDatanodeDetails.randomDatanodeDetails(); + dnList.add(dn3); + UUID dnId3 = dn3.getUuid(); + List reports3 = generateStorageReportProto( + volumeCount, dnId3, 1000, 1001); + nodeManager.register(dn3, HddsTestUtils.createNodeReport(reports3, + emptyList()), null); + nodeManager.processHeartbeat(dn3, layoutInfo); + // Generate the 4th DataNode + DatanodeDetails dn4 = MockDatanodeDetails.randomDatanodeDetails(); + dnList.add(dn4); + UUID dnId4 = dn4.getUuid(); + nodeManager.register(dn4, null, null); + nodeManager.processHeartbeat(dn4, layoutInfo); + + eventQueue.processAll(8000L); + NodeStateManager nodeStateManager = nodeManager.getNodeStateManager(); + List dataNodeInfoList = nodeStateManager.getAllNodes(); + for (DatanodeInfo datanodeInfo : dataNodeInfoList) { + double usedPerc = nodeManager.calculateStoragePercentage(datanodeInfo); + if (datanodeInfo.getUuid().toString().equals(dnId1.toString())) { + assertEquals(usedPerc, 60); + } else if (datanodeInfo.getUuid().toString().equals(dnId2.toString())) { + assertEquals(usedPerc, 100); + } else if (datanodeInfo.getUuid().toString().equals(dnId3.toString())) { + assertEquals(usedPerc, 100); + } else if (datanodeInfo.getUuid().toString().equals(dnId4.toString())) { + assertEquals(usedPerc, 0); + } + } + dnList.clear(); + } + } + /** * Test multiple nodes sending initial heartbeat with their node report * with multiple volumes. From 0a40dd1c822dea13304a4ff246e520e56bd66091 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 7 Jan 2024 02:01:51 +0800 Subject: [PATCH 2/8] Fix findbugs --- .../org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index e1f9a83e554..9e3f03d9edf 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -1593,7 +1593,7 @@ public void testCalculateStoragePercentage() OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, MILLISECONDS); - List dnList = new ArrayList<>(4); + List dnList = new ArrayList<>(); try (SCMNodeManager nodeManager = createNodeManager(conf)) { EventQueue eventQueue = (EventQueue) scm.getEventQueue(); LayoutVersionManager versionManager = From a1f4c7104a451b50c5507cea89c9a22168043f7a Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sun, 7 Jan 2024 08:05:16 +0800 Subject: [PATCH 3/8] Fix findbugs --- .../org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 9e3f03d9edf..bf0ea236125 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -1593,7 +1593,6 @@ public void testCalculateStoragePercentage() OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, MILLISECONDS); - List dnList = new ArrayList<>(); try (SCMNodeManager nodeManager = createNodeManager(conf)) { EventQueue eventQueue = (EventQueue) scm.getEventQueue(); LayoutVersionManager versionManager = @@ -1603,7 +1602,6 @@ public void testCalculateStoragePercentage() versionManager.getSoftwareLayoutVersion()); // Generate the first DataNode DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); - dnList.add(dn1); UUID dnId1 = dn1.getUuid(); int volumeCount = 10; List reports1 = generateStorageReportProto( @@ -1613,7 +1611,6 @@ public void testCalculateStoragePercentage() nodeManager.processHeartbeat(dn1, layoutInfo); // Generate the 2nd DataNode DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails(); - dnList.add(dn2); UUID dnId2 = dn2.getUuid(); List reports2 = generateStorageReportProto( volumeCount, dnId2, 1000, 1000); @@ -1622,7 +1619,6 @@ public void testCalculateStoragePercentage() nodeManager.processHeartbeat(dn2, layoutInfo); // Generate the 3rd DataNode DatanodeDetails dn3 = MockDatanodeDetails.randomDatanodeDetails(); - dnList.add(dn3); UUID dnId3 = dn3.getUuid(); List reports3 = generateStorageReportProto( volumeCount, dnId3, 1000, 1001); @@ -1631,7 +1627,6 @@ public void testCalculateStoragePercentage() nodeManager.processHeartbeat(dn3, layoutInfo); // Generate the 4th DataNode DatanodeDetails dn4 = MockDatanodeDetails.randomDatanodeDetails(); - dnList.add(dn4); UUID dnId4 = dn4.getUuid(); nodeManager.register(dn4, null, null); nodeManager.processHeartbeat(dn4, layoutInfo); @@ -1651,7 +1646,6 @@ public void testCalculateStoragePercentage() assertEquals(usedPerc, 0); } } - dnList.clear(); } } From a1a969b3c7a9c5670170fe20a416f82ab1bb84ac Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Tue, 9 Jan 2024 20:45:36 +0800 Subject: [PATCH 4/8] Improve some unit tests. --- .../hadoop/hdds/scm/node/SCMNodeManager.java | 4 +- .../resources/webapps/scm/scm-overview.html | 6 +- .../src/main/resources/webapps/scm/scm.js | 2 +- .../hdds/scm/node/TestSCMNodeManager.java | 74 +++++++------------ 4 files changed, 34 insertions(+), 52 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 68cad9f7dc8..1aff26104f8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -143,7 +143,7 @@ public class SCMNodeManager implements NodeManager { private final String opeState = "OPSTATE"; private final String comState = "COMSTATE"; private final String lastHeartbeat = "LASTHEARTBEAT"; - private final String usedPercent = "USEDPERCENT"; + private final String usedSpacePercent = "USEDSPACEPERCENT"; /** * Constructs SCM machine Manager. */ @@ -1117,7 +1117,7 @@ public Map> getNodeStatusInfo() { httpsPort.getValue().toString()); } double usedPec = calculateStoragePercentage(dni); - map.put(usedPercent, usedPec + "%"); + map.put(usedSpacePercent, usedPec + "%"); nodes.put(hostName, map); } return nodes; diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index 8e4d00ea9e3..cb7cd4c2dc3 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -48,8 +48,8 @@

Node Status

HostName - Used Percent + Used Space Percent Operational State element.key === "USEDPERCENT").value, + usedspacepercent: value && value.find((element) => element.key === "USEDSPACEPERCENT").value, comstate: value && value.find((element) => element.key === "COMSTATE").value, lastheartbeat: value && value.find((element) => element.key === "LASTHEARTBEAT").value, port: portSpec.port, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index bf0ea236125..8d5db45a289 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -86,6 +86,7 @@ import java.util.Map; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -123,6 +124,8 @@ import static org.mockito.Mockito.eq; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; @@ -1587,9 +1590,21 @@ private List generateStorageReportProto( return reports; } - @Test - public void testCalculateStoragePercentage() - throws IOException, AuthenticationException { + private static Stream calculateStoragePercentageScenarios() { + return Stream.of( + Arguments.of(1000, 600, 10, 60), + Arguments.of(1000, 1000, 10, 100), + Arguments.of(1000, 1001, 10, 100), + Arguments.of(0, 0, 0, 0), + Arguments.of(1010, 547, 5, 54.16) + ); + } + + @ParameterizedTest + @MethodSource("calculateStoragePercentageScenarios") + public void testCalculateStoragePercentage(long capacity, + long used, int volumeCount, double targetUsedPerc) + throws AuthenticationException, IOException { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, MILLISECONDS); @@ -1600,52 +1615,19 @@ public void testCalculateStoragePercentage() LayoutVersionProto layoutInfo = toLayoutVersionProto( versionManager.getMetadataLayoutVersion(), versionManager.getSoftwareLayoutVersion()); - // Generate the first DataNode - DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); - UUID dnId1 = dn1.getUuid(); - int volumeCount = 10; - List reports1 = generateStorageReportProto( - volumeCount, dnId1, 1000, 600); - nodeManager.register(dn1, HddsTestUtils.createNodeReport(reports1, - emptyList()), null); - nodeManager.processHeartbeat(dn1, layoutInfo); - // Generate the 2nd DataNode - DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails(); - UUID dnId2 = dn2.getUuid(); - List reports2 = generateStorageReportProto( - volumeCount, dnId2, 1000, 1000); - nodeManager.register(dn2, HddsTestUtils.createNodeReport(reports2, - emptyList()), null); - nodeManager.processHeartbeat(dn2, layoutInfo); - // Generate the 3rd DataNode - DatanodeDetails dn3 = MockDatanodeDetails.randomDatanodeDetails(); - UUID dnId3 = dn3.getUuid(); - List reports3 = generateStorageReportProto( - volumeCount, dnId3, 1000, 1001); - nodeManager.register(dn3, HddsTestUtils.createNodeReport(reports3, - emptyList()), null); - nodeManager.processHeartbeat(dn3, layoutInfo); - // Generate the 4th DataNode - DatanodeDetails dn4 = MockDatanodeDetails.randomDatanodeDetails(); - UUID dnId4 = dn4.getUuid(); - nodeManager.register(dn4, null, null); - nodeManager.processHeartbeat(dn4, layoutInfo); - + DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + UUID dnId = dn.getUuid(); + List reports = volumeCount > 0 ? + generateStorageReportProto(volumeCount, dnId, capacity, used) : null; + nodeManager.register(dn, reports != null ? + HddsTestUtils.createNodeReport(reports, emptyList()) : null, null); + nodeManager.processHeartbeat(dn, layoutInfo); eventQueue.processAll(8000L); NodeStateManager nodeStateManager = nodeManager.getNodeStateManager(); List dataNodeInfoList = nodeStateManager.getAllNodes(); - for (DatanodeInfo datanodeInfo : dataNodeInfoList) { - double usedPerc = nodeManager.calculateStoragePercentage(datanodeInfo); - if (datanodeInfo.getUuid().toString().equals(dnId1.toString())) { - assertEquals(usedPerc, 60); - } else if (datanodeInfo.getUuid().toString().equals(dnId2.toString())) { - assertEquals(usedPerc, 100); - } else if (datanodeInfo.getUuid().toString().equals(dnId3.toString())) { - assertEquals(usedPerc, 100); - } else if (datanodeInfo.getUuid().toString().equals(dnId4.toString())) { - assertEquals(usedPerc, 0); - } - } + double usedPerc = nodeManager.calculateStoragePercentage( + dataNodeInfoList.get(0)); + assertEquals(usedPerc, targetUsedPerc); } } From 35629206537f25f4fc194f55f3e9acb11f0eb446 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Tue, 9 Jan 2024 23:00:39 +0800 Subject: [PATCH 5/8] Improve some unit tests. --- .../hadoop/hdds/scm/node/SCMNodeManager.java | 46 ++++++++++++++++++- .../resources/webapps/scm/scm-overview.html | 3 ++ .../src/main/resources/webapps/scm/scm.js | 1 + .../hdds/scm/node/TestSCMNodeManager.java | 40 ++++++++++++++++ 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 1aff26104f8..2ddf9f9e252 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -144,6 +144,7 @@ public class SCMNodeManager implements NodeManager { private final String comState = "COMSTATE"; private final String lastHeartbeat = "LASTHEARTBEAT"; private final String usedSpacePercent = "USEDSPACEPERCENT"; + private final String totalCapacity = "CAPACITY"; /** * Constructs SCM machine Manager. */ @@ -1116,6 +1117,8 @@ public Map> getNodeStatusInfo() { map.put(httpsPort.getName().toString(), httpsPort.getValue().toString()); } + String capacity = calculateStorageCapacity(dni); + map.put(totalCapacity, capacity); double usedPec = calculateStoragePercentage(dni); map.put(usedSpacePercent, usedPec + "%"); nodes.put(hostName, map); @@ -1123,6 +1126,45 @@ public Map> getNodeStatusInfo() { return nodes; } + /** + * Calculate the storage capacity of the DataNode node. + * @param dni DataNode node that needs to be calculated. + * @return + */ + public String calculateStorageCapacity(DatanodeInfo dni) { + long capacityByte = 0; + List storageReports = dni.getStorageReports(); + if (storageReports != null && !storageReports.isEmpty()) { + for (StorageReportProto storageReport : storageReports) { + capacityByte += storageReport.getCapacity(); + } + } + + double ua = capacityByte; + String unit = "B"; + if (ua > 1024) { + ua = ua / 1024; + unit = "KB"; + } + if (ua > 1024) { + ua = ua / 1024; + unit = "MB"; + } + if (ua > 1024) { + ua = ua / 1024; + unit = "GB"; + } + if (ua > 1024) { + ua = ua / 1024; + unit = "TB"; + } + + DecimalFormat decimalFormat = new DecimalFormat("#0.0"); + decimalFormat.setRoundingMode(RoundingMode.HALF_UP); + double capacity = Double.valueOf(decimalFormat.format(ua)); + return capacity + unit; + } + /** * Calculate the storage usage percentage of a DataNode node. * @param dni DataNode node that needs to be calculated @@ -1135,8 +1177,8 @@ public double calculateStoragePercentage(DatanodeInfo dni) { long capacity = 0; long scmUsed = 0; for (StorageReportProto storageReport : storageReports) { - capacity = capacity + storageReport.getCapacity(); - scmUsed = scmUsed + storageReport.getScmUsed(); + capacity += storageReport.getCapacity(); + scmUsed += storageReport.getScmUsed(); } double percent = (double) scmUsed / capacity; percent = percent > 1.0 ? 1.0 : percent; diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index cb7cd4c2dc3..fdd8de15b6a 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -50,6 +50,8 @@

Node Status

'sortdesc':(columnName == 'hostname' && !reverse)}">HostName Used Space Percent + Capacity Operational State element.key === "USEDSPACEPERCENT").value, + capacity: value && value.find((element) => element.key === "CAPACITY").value, comstate: value && value.find((element) => element.key === "COMSTATE").value, lastheartbeat: value && value.find((element) => element.key === "LASTHEARTBEAT").value, port: portSpec.port, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 8d5db45a289..8fbab3645db 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -1631,6 +1631,46 @@ public void testCalculateStoragePercentage(long capacity, } } + private static Stream calculateStorageCapacityScenarios() { + return Stream.of( + Arguments.of(600, 6, 1, "600.0B"), + Arguments.of(10000, 1000, 12, "117.2KB"), + Arguments.of(100000000, 1000, 12, "1.1GB"), + Arguments.of(10000, 1000, 0, "0.0B") + ); + } + + @ParameterizedTest + @MethodSource("calculateStorageCapacityScenarios") + public void testcalculateStorageCapacity(long capacity, + long used, int volumeCount, String totalCapacity) + throws AuthenticationException, IOException { + OzoneConfiguration conf = getConf(); + conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, + MILLISECONDS); + try (SCMNodeManager nodeManager = createNodeManager(conf)) { + EventQueue eventQueue = (EventQueue) scm.getEventQueue(); + LayoutVersionManager versionManager = + nodeManager.getLayoutVersionManager(); + LayoutVersionProto layoutInfo = toLayoutVersionProto( + versionManager.getMetadataLayoutVersion(), + versionManager.getSoftwareLayoutVersion()); + DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + UUID dnId = dn.getUuid(); + List reports = volumeCount > 0 ? + generateStorageReportProto(volumeCount, dnId, capacity, used) : null; + nodeManager.register(dn, reports != null ? + HddsTestUtils.createNodeReport(reports, emptyList()) : null, null); + nodeManager.processHeartbeat(dn, layoutInfo); + eventQueue.processAll(8000L); + NodeStateManager nodeStateManager = nodeManager.getNodeStateManager(); + List dataNodeInfoList = nodeStateManager.getAllNodes(); + String capacityResult = nodeManager.calculateStorageCapacity( + dataNodeInfoList.get(0)); + assertEquals(capacityResult, totalCapacity); + } + } + /** * Test multiple nodes sending initial heartbeat with their node report * with multiple volumes. From 31f2b8708a21dc03a69b0ae67256aec0aa9c8cb0 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sat, 13 Jan 2024 01:48:45 +0800 Subject: [PATCH 6/8] Improve some unit tests. --- .../hadoop/hdds/scm/node/SCMNodeManager.java | 48 ++++++++----- .../resources/webapps/scm/scm-overview.html | 2 +- .../hdds/scm/node/TestSCMNodeManager.java | 71 +++++-------------- 3 files changed, 51 insertions(+), 70 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 2ddf9f9e252..17041b7d7ac 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -1119,8 +1119,11 @@ public Map> getNodeStatusInfo() { } String capacity = calculateStorageCapacity(dni); map.put(totalCapacity, capacity); - double usedPec = calculateStoragePercentage(dni); - map.put(usedSpacePercent, usedPec + "%"); + double[] storagePercentage = calculateStoragePercentage(dni); + double scmUsedPerc = storagePercentage[0]; + double nonScmUsedPerc = storagePercentage[1]; + map.put(usedSpacePercent, + "scmUsed: " + scmUsedPerc + "%, nonScmUsed: " + nonScmUsedPerc + "%"); nodes.put(hostName, map); } return nodes; @@ -1141,52 +1144,65 @@ public String calculateStorageCapacity(DatanodeInfo dni) { } double ua = capacityByte; - String unit = "B"; + StringBuilder unit = new StringBuilder("B"); if (ua > 1024) { ua = ua / 1024; - unit = "KB"; + unit.replace(0, 1, "KB"); } if (ua > 1024) { ua = ua / 1024; - unit = "MB"; + unit.replace(0, 2, "MB"); } if (ua > 1024) { ua = ua / 1024; - unit = "GB"; + unit.replace(0, 2, "GB"); } if (ua > 1024) { ua = ua / 1024; - unit = "TB"; + unit.replace(0, 2, "TB"); } DecimalFormat decimalFormat = new DecimalFormat("#0.0"); decimalFormat.setRoundingMode(RoundingMode.HALF_UP); double capacity = Double.valueOf(decimalFormat.format(ua)); - return capacity + unit; + return capacity + unit.toString(); } /** * Calculate the storage usage percentage of a DataNode node. - * @param dni DataNode node that needs to be calculated + * @param dni DataNode node that needs to be calculated. * @return */ - public double calculateStoragePercentage(DatanodeInfo dni) { - double usedPec = 0; + public double[] calculateStoragePercentage(DatanodeInfo dni) { + double[] storagePercentage = new double[2]; + double usedPercentage = 0; + double nonUsedPercentage = 0; List storageReports = dni.getStorageReports(); if (storageReports != null && !storageReports.isEmpty()) { long capacity = 0; long scmUsed = 0; + long remaining = 0; for (StorageReportProto storageReport : storageReports) { capacity += storageReport.getCapacity(); scmUsed += storageReport.getScmUsed(); + remaining += storageReport.getRemaining(); } - double percent = (double) scmUsed / capacity; - percent = percent > 1.0 ? 1.0 : percent; - DecimalFormat decimalFormat = new DecimalFormat("#0.0000"); + long scmNonUsed = capacity - scmUsed - remaining; + + DecimalFormat decimalFormat = new DecimalFormat("#0.00"); decimalFormat.setRoundingMode(RoundingMode.HALF_UP); - usedPec = Double.valueOf(decimalFormat.format(percent)) * 100; + + double usedPerc = ((double) scmUsed / capacity) * 100; + usedPerc = usedPerc > 100.0 ? 100.0 : usedPerc; + double nonUsedPerc = ((double) scmNonUsed / capacity) * 100; + nonUsedPerc = nonUsedPerc > 100.0 ? 100.0 : nonUsedPerc; + usedPercentage = Double.valueOf(decimalFormat.format(usedPerc)); + nonUsedPercentage = Double.valueOf(decimalFormat.format(nonUsedPerc)); } - return usedPec; + + storagePercentage[0] = usedPercentage; + storagePercentage[1] = nonUsedPercentage; + return storagePercentage; } /** diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index fdd8de15b6a..ceb65d0f3f3 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -51,7 +51,7 @@

Node Status

Used Space Percent Capacity + 'sortdesc':(columnName == 'capacity' && !reverse)}">Total Operational State generateStorageReportProto( private static Stream calculateStoragePercentageScenarios() { return Stream.of( - Arguments.of(1000, 600, 10, 60), - Arguments.of(1000, 1000, 10, 100), - Arguments.of(1000, 1001, 10, 100), - Arguments.of(0, 0, 0, 0), - Arguments.of(1010, 547, 5, 54.16) + Arguments.of(600, 65, 500, 1, "600.0B", 10.83, 5.83), + Arguments.of(10000, 1000, 8800, 12, "117.2KB", 10.0, 2.0), + Arguments.of(100000000, 1000, 899999, 12, "1.1GB", 0.0, 99.1), + Arguments.of(10000, 1000, 0, 0, "0.0B", 0.0, 0.0), + Arguments.of(0, 0, 0, 0, "0.0B", 0.0, 0.0), + Arguments.of(1010, 547, 400, 5, "4.9KB", 54.16, 6.24) ); } @ParameterizedTest @MethodSource("calculateStoragePercentageScenarios") - public void testCalculateStoragePercentage(long capacity, - long used, int volumeCount, double targetUsedPerc) - throws AuthenticationException, IOException { - OzoneConfiguration conf = getConf(); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, - MILLISECONDS); - try (SCMNodeManager nodeManager = createNodeManager(conf)) { - EventQueue eventQueue = (EventQueue) scm.getEventQueue(); - LayoutVersionManager versionManager = - nodeManager.getLayoutVersionManager(); - LayoutVersionProto layoutInfo = toLayoutVersionProto( - versionManager.getMetadataLayoutVersion(), - versionManager.getSoftwareLayoutVersion()); - DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); - UUID dnId = dn.getUuid(); - List reports = volumeCount > 0 ? - generateStorageReportProto(volumeCount, dnId, capacity, used) : null; - nodeManager.register(dn, reports != null ? - HddsTestUtils.createNodeReport(reports, emptyList()) : null, null); - nodeManager.processHeartbeat(dn, layoutInfo); - eventQueue.processAll(8000L); - NodeStateManager nodeStateManager = nodeManager.getNodeStateManager(); - List dataNodeInfoList = nodeStateManager.getAllNodes(); - double usedPerc = nodeManager.calculateStoragePercentage( - dataNodeInfoList.get(0)); - assertEquals(usedPerc, targetUsedPerc); - } - } - - private static Stream calculateStorageCapacityScenarios() { - return Stream.of( - Arguments.of(600, 6, 1, "600.0B"), - Arguments.of(10000, 1000, 12, "117.2KB"), - Arguments.of(100000000, 1000, 12, "1.1GB"), - Arguments.of(10000, 1000, 0, "0.0B") - ); - } - - @ParameterizedTest - @MethodSource("calculateStorageCapacityScenarios") - public void testcalculateStorageCapacity(long capacity, - long used, int volumeCount, String totalCapacity) + public void testCalculateStoragePercentage(long perCapacity, + long used, long remaining, int volumeCount, String totalCapacity, + double scmUsedPerc, double nonScmUsedPerc) throws AuthenticationException, IOException { OzoneConfiguration conf = getConf(); conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, @@ -1658,16 +1619,20 @@ public void testcalculateStorageCapacity(long capacity, DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); UUID dnId = dn.getUuid(); List reports = volumeCount > 0 ? - generateStorageReportProto(volumeCount, dnId, capacity, used) : null; + generateStorageReportProto(volumeCount, dnId, perCapacity, + used, remaining) : null; nodeManager.register(dn, reports != null ? HddsTestUtils.createNodeReport(reports, emptyList()) : null, null); nodeManager.processHeartbeat(dn, layoutInfo); eventQueue.processAll(8000L); NodeStateManager nodeStateManager = nodeManager.getNodeStateManager(); List dataNodeInfoList = nodeStateManager.getAllNodes(); - String capacityResult = nodeManager.calculateStorageCapacity( - dataNodeInfoList.get(0)); + DatanodeInfo dni = dataNodeInfoList.get(0); + String capacityResult = nodeManager.calculateStorageCapacity(dni); assertEquals(capacityResult, totalCapacity); + double[] storagePercentage = nodeManager.calculateStoragePercentage(dni); + assertEquals(storagePercentage[0], scmUsedPerc); + assertEquals(storagePercentage[1], nonScmUsedPerc); } } From 403e2e2883096503bc140fd8c49c93f4bf297a1c Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Sat, 13 Jan 2024 22:02:00 +0800 Subject: [PATCH 7/8] Improve some UI display styles. --- .../server-scm/src/main/resources/webapps/scm/scm-overview.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html index ceb65d0f3f3..fdd8de15b6a 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html @@ -51,7 +51,7 @@

Node Status

Used Space Percent Total + 'sortdesc':(columnName == 'capacity' && !reverse)}">Capacity Operational State > getNodeStatusInfo() { heartbeatTimeDiff = getLastHeartbeatTimeDiff(dni.getLastHeartbeatTime()); } Map map = new HashMap<>(); - map.put(opeState, opstate); - map.put(comState, healthState); - map.put(lastHeartbeat, heartbeatTimeDiff); + map.put(OPESTATE, opstate); + map.put(COMSTATE, healthState); + map.put(LASTHEARTBEAT, heartbeatTimeDiff); if (httpPort != null) { map.put(httpPort.getName().toString(), httpPort.getValue().toString()); } @@ -1117,13 +1117,14 @@ public Map> getNodeStatusInfo() { map.put(httpsPort.getName().toString(), httpsPort.getValue().toString()); } - String capacity = calculateStorageCapacity(dni); - map.put(totalCapacity, capacity); - double[] storagePercentage = calculateStoragePercentage(dni); - double scmUsedPerc = storagePercentage[0]; - double nonScmUsedPerc = storagePercentage[1]; - map.put(usedSpacePercent, - "scmUsed: " + scmUsedPerc + "%, nonScmUsed: " + nonScmUsedPerc + "%"); + String capacity = calculateStorageCapacity(dni.getStorageReports()); + map.put(TOTALCAPACITY, capacity); + String[] storagePercentage = calculateStoragePercentage( + dni.getStorageReports()); + String scmUsedPerc = storagePercentage[0]; + String nonScmUsedPerc = storagePercentage[1]; + map.put(USEDSPACEPERCENT, + "Ozone: " + scmUsedPerc + "%, other: " + nonScmUsedPerc + "%"); nodes.put(hostName, map); } return nodes; @@ -1131,12 +1132,13 @@ public Map> getNodeStatusInfo() { /** * Calculate the storage capacity of the DataNode node. - * @param dni DataNode node that needs to be calculated. + * @param storageReports Calculate the storage capacity corresponding + * to the storage collection. * @return */ - public String calculateStorageCapacity(DatanodeInfo dni) { + public static String calculateStorageCapacity( + List storageReports) { long capacityByte = 0; - List storageReports = dni.getStorageReports(); if (storageReports != null && !storageReports.isEmpty()) { for (StorageReportProto storageReport : storageReports) { capacityByte += storageReport.getCapacity(); @@ -1164,20 +1166,21 @@ public String calculateStorageCapacity(DatanodeInfo dni) { DecimalFormat decimalFormat = new DecimalFormat("#0.0"); decimalFormat.setRoundingMode(RoundingMode.HALF_UP); - double capacity = Double.valueOf(decimalFormat.format(ua)); + String capacity = decimalFormat.format(ua); return capacity + unit.toString(); } /** * Calculate the storage usage percentage of a DataNode node. - * @param dni DataNode node that needs to be calculated. + * @param storageReports Calculate the storage percentage corresponding + * to the storage collection. * @return */ - public double[] calculateStoragePercentage(DatanodeInfo dni) { - double[] storagePercentage = new double[2]; - double usedPercentage = 0; - double nonUsedPercentage = 0; - List storageReports = dni.getStorageReports(); + public static String[] calculateStoragePercentage( + List storageReports) { + String[] storagePercentage = new String[2]; + String usedPercentage = "N/A"; + String nonUsedPercentage = "N/A"; if (storageReports != null && !storageReports.isEmpty()) { long capacity = 0; long scmUsed = 0; @@ -1196,8 +1199,8 @@ public double[] calculateStoragePercentage(DatanodeInfo dni) { usedPerc = usedPerc > 100.0 ? 100.0 : usedPerc; double nonUsedPerc = ((double) scmNonUsed / capacity) * 100; nonUsedPerc = nonUsedPerc > 100.0 ? 100.0 : nonUsedPerc; - usedPercentage = Double.valueOf(decimalFormat.format(usedPerc)); - nonUsedPercentage = Double.valueOf(decimalFormat.format(nonUsedPerc)); + usedPercentage = decimalFormat.format(usedPerc); + nonUsedPercentage = decimalFormat.format(nonUsedPerc); } storagePercentage[0] = usedPercentage; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 82e6bdf81d3..930774a54bf 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -1591,12 +1591,12 @@ private List generateStorageReportProto( private static Stream calculateStoragePercentageScenarios() { return Stream.of( - Arguments.of(600, 65, 500, 1, "600.0B", 10.83, 5.83), - Arguments.of(10000, 1000, 8800, 12, "117.2KB", 10.0, 2.0), - Arguments.of(100000000, 1000, 899999, 12, "1.1GB", 0.0, 99.1), - Arguments.of(10000, 1000, 0, 0, "0.0B", 0.0, 0.0), - Arguments.of(0, 0, 0, 0, "0.0B", 0.0, 0.0), - Arguments.of(1010, 547, 400, 5, "4.9KB", 54.16, 6.24) + Arguments.of(600, 65, 500, 1, "600.0B", "10.83", "5.83"), + Arguments.of(10000, 1000, 8800, 12, "117.2KB", "10.00", "2.00"), + Arguments.of(100000000, 1000, 899999, 12, "1.1GB", "0.00", "99.10"), + Arguments.of(10000, 1000, 0, 0, "0.0B", "N/A", "N/A"), + Arguments.of(0, 0, 0, 0, "0.0B", "N/A", "N/A"), + Arguments.of(1010, 547, 400, 5, "4.9KB", "54.16", "6.24") ); } @@ -1604,36 +1604,18 @@ private static Stream calculateStoragePercentageScenarios() { @MethodSource("calculateStoragePercentageScenarios") public void testCalculateStoragePercentage(long perCapacity, long used, long remaining, int volumeCount, String totalCapacity, - double scmUsedPerc, double nonScmUsedPerc) - throws AuthenticationException, IOException { - OzoneConfiguration conf = getConf(); - conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000, - MILLISECONDS); - try (SCMNodeManager nodeManager = createNodeManager(conf)) { - EventQueue eventQueue = (EventQueue) scm.getEventQueue(); - LayoutVersionManager versionManager = - nodeManager.getLayoutVersionManager(); - LayoutVersionProto layoutInfo = toLayoutVersionProto( - versionManager.getMetadataLayoutVersion(), - versionManager.getSoftwareLayoutVersion()); - DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); - UUID dnId = dn.getUuid(); - List reports = volumeCount > 0 ? - generateStorageReportProto(volumeCount, dnId, perCapacity, - used, remaining) : null; - nodeManager.register(dn, reports != null ? - HddsTestUtils.createNodeReport(reports, emptyList()) : null, null); - nodeManager.processHeartbeat(dn, layoutInfo); - eventQueue.processAll(8000L); - NodeStateManager nodeStateManager = nodeManager.getNodeStateManager(); - List dataNodeInfoList = nodeStateManager.getAllNodes(); - DatanodeInfo dni = dataNodeInfoList.get(0); - String capacityResult = nodeManager.calculateStorageCapacity(dni); - assertEquals(capacityResult, totalCapacity); - double[] storagePercentage = nodeManager.calculateStoragePercentage(dni); - assertEquals(storagePercentage[0], scmUsedPerc); - assertEquals(storagePercentage[1], nonScmUsedPerc); - } + String scmUsedPerc, String nonScmUsedPerc) { + DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + UUID dnId = dn.getUuid(); + List reports = volumeCount > 0 ? + generateStorageReportProto(volumeCount, dnId, perCapacity, + used, remaining) : null; + String capacityResult = SCMNodeManager.calculateStorageCapacity(reports); + assertEquals(totalCapacity, capacityResult); + String[] storagePercentage = SCMNodeManager.calculateStoragePercentage( + reports); + assertEquals(scmUsedPerc, storagePercentage[0]); + assertEquals(nonScmUsedPerc, storagePercentage[1]); } /**