diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java index fe0c29e510fa..0bfedd43960f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java @@ -75,7 +75,6 @@ public class ContainerBalancerTask implements Runnable { private OzoneConfiguration ozoneConfiguration; private ContainerBalancer containerBalancer; private final SCMContext scmContext; - private double threshold; private int totalNodesInCluster; private double maxDatanodesRatioToInvolvePerIteration; private long maxSizeToMovePerIteration; @@ -84,17 +83,13 @@ public class ContainerBalancerTask implements Runnable { // count actual size moved in bytes private long sizeActuallyMovedInLatestIteration; private int iterations; - private List unBalancedNodes; - private List overUtilizedNodes; - private List underUtilizedNodes; + private final List overUtilizedNodes; + private final List underUtilizedNodes; private List withinThresholdUtilizedNodes; private Set excludeNodes; private Set includeNodes; private ContainerBalancerConfiguration config; private ContainerBalancerMetrics metrics; - private long clusterCapacity; - private long clusterRemaining; - private double clusterAvgUtilisation; private PlacementPolicyValidateProxy placementPolicyValidateProxy; private NetworkTopology networkTopology; private double upperLimit; @@ -150,7 +145,6 @@ public ContainerBalancerTask(StorageContainerManager scm, this.overUtilizedNodes = new ArrayList<>(); this.underUtilizedNodes = new ArrayList<>(); this.withinThresholdUtilizedNodes = new ArrayList<>(); - this.unBalancedNodes = new ArrayList<>(); this.placementPolicyValidateProxy = scm.getPlacementPolicyValidateProxy(); this.networkTopology = scm.getClusterMap(); this.nextIterationIndex = nextIterationIndex; @@ -348,7 +342,6 @@ private boolean initializeIteration() { return false; } - this.threshold = config.getThresholdAsRatio(); this.maxDatanodesRatioToInvolvePerIteration = config.getMaxDatanodesRatioToInvolvePerIteration(); this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration(); @@ -368,22 +361,19 @@ private boolean initializeIteration() { this.totalNodesInCluster = datanodeUsageInfos.size(); - clusterAvgUtilisation = calculateAvgUtilization(datanodeUsageInfos); + double clusterAvgUtilisation = calculateAvgUtilization(datanodeUsageInfos); if (LOG.isDebugEnabled()) { - LOG.debug("Average utilization of the cluster is {}", - clusterAvgUtilisation); + LOG.debug("Average utilization of the cluster is {}", clusterAvgUtilisation); } - // over utilized nodes have utilization(that is, used / capacity) greater - // than upper limit + double threshold = config.getThresholdAsRatio(); + // over utilized nodes have utilization(that is, used / capacity) greater than upper limit this.upperLimit = clusterAvgUtilisation + threshold; - // under utilized nodes have utilization(that is, used / capacity) less - // than lower limit + // under utilized nodes have utilization(that is, used / capacity) less than lower limit this.lowerLimit = clusterAvgUtilisation - threshold; if (LOG.isDebugEnabled()) { - LOG.debug("Lower limit for utilization is {} and Upper limit for " + - "utilization is {}", lowerLimit, upperLimit); + LOG.debug("Lower limit for utilization is {} and Upper limit for utilization is {}", lowerLimit, upperLimit); } long totalOverUtilizedBytes = 0L, totalUnderUtilizedBytes = 0L; @@ -433,12 +423,7 @@ private boolean initializeIteration() { OzoneConsts.GB); Collections.reverse(underUtilizedNodes); - unBalancedNodes = new ArrayList<>( - overUtilizedNodes.size() + underUtilizedNodes.size()); - unBalancedNodes.addAll(overUtilizedNodes); - unBalancedNodes.addAll(underUtilizedNodes); - - if (unBalancedNodes.isEmpty()) { + if (overUtilizedNodes.isEmpty() && underUtilizedNodes.isEmpty()) { LOG.info("Did not find any unbalanced Datanodes."); return false; } @@ -487,7 +472,7 @@ private IterationResult doIteration() { findTargetStrategy.reInitialize(potentialTargets, config, upperLimit); findSourceStrategy.reInitialize(getPotentialSources(), config, lowerLimit); - moveSelectionToFutureMap = new HashMap<>(unBalancedNodes.size()); + moveSelectionToFutureMap = new HashMap<>(underUtilizedNodes.size() + overUtilizedNodes.size()); boolean isMoveGeneratedInThisIteration = false; iterationResult = IterationResult.ITERATION_COMPLETED; boolean canAdaptWhenNearingLimits = true; @@ -965,8 +950,8 @@ private long ratioToBytes(Long nodeCapacity, double utilizationRatio) { * @return Average utilization value */ @VisibleForTesting - double calculateAvgUtilization(List nodes) { - if (nodes.size() == 0) { + public static double calculateAvgUtilization(List nodes) { + if (nodes.isEmpty()) { LOG.warn("No nodes to calculate average utilization for in " + "ContainerBalancer."); return 0; @@ -976,8 +961,8 @@ private long ratioToBytes(Long nodeCapacity, double utilizationRatio) { for (DatanodeUsageInfo node : nodes) { aggregatedStats.add(node.getScmNodeStat()); } - clusterCapacity = aggregatedStats.getCapacity().get(); - clusterRemaining = aggregatedStats.getRemaining().get(); + long clusterCapacity = aggregatedStats.getCapacity().get(); + long clusterRemaining = aggregatedStats.getRemaining().get(); return (clusterCapacity - clusterRemaining) / (double) clusterCapacity; } @@ -1060,11 +1045,8 @@ private void incSizeSelectedForMoving(DatanodeDetails source, */ private void resetState() { moveManager.resetState(); - this.clusterCapacity = 0L; - this.clusterRemaining = 0L; this.overUtilizedNodes.clear(); this.underUtilizedNodes.clear(); - this.unBalancedNodes.clear(); this.containerToSourceMap.clear(); this.containerToTargetMap.clear(); this.selectedSources.clear(); @@ -1090,15 +1072,14 @@ private boolean isBalancerRunning() { return taskStatus == Status.RUNNING; } - /** - * Gets the list of unBalanced nodes, that is, the over and under utilized - * nodes in the cluster. - * - * @return List of DatanodeUsageInfo containing unBalanced nodes. - */ @VisibleForTesting - List getUnBalancedNodes() { - return unBalancedNodes; + public List getOverUtilizedNodes() { + return overUtilizedNodes; + } + + @VisibleForTesting + public List getUnderUtilizedNodes() { + return underUtilizedNodes; } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/MockedSCM.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/MockedSCM.java index b290b950442b..a3ec55d58639 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/MockedSCM.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/MockedSCM.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -66,28 +66,32 @@ public final class MockedSCM { private final StorageContainerManager scm; private final TestableCluster cluster; private final MockNodeManager mockNodeManager; - private MockedReplicationManager mockedReplicaManager; - private MoveManager moveManager; - private ContainerManager containerManager; - + private final MockedReplicationManager mockedReplicaManager; + private final MoveManager moveManager; + private final ContainerManager containerManager; private MockedPlacementPolicies mockedPlacementPolicies; public MockedSCM(@Nonnull TestableCluster testableCluster) { scm = mock(StorageContainerManager.class); cluster = testableCluster; mockNodeManager = new MockNodeManager(cluster.getDatanodeToContainersMap()); + try { + moveManager = mockMoveManager(); + containerManager = mockContainerManager(cluster); + mockedReplicaManager = MockedReplicationManager.doMock(); + } catch (NodeNotFoundException | ContainerReplicaNotFoundException | ContainerNotFoundException | + TimeoutException e + ) { + throw new RuntimeException("Can't create MockedSCM instance: ", e); + } } - public void init(@Nonnull ContainerBalancerConfiguration balancerConfig) { - init(balancerConfig, new OzoneConfiguration()); - } - - public void init(@Nonnull ContainerBalancerConfiguration balancerConfig, @Nonnull OzoneConfiguration ozoneCfg) { + private void init(@Nonnull ContainerBalancerConfiguration balancerConfig, @Nonnull OzoneConfiguration ozoneCfg) { ozoneCfg.setFromObject(balancerConfig); try { doMock(balancerConfig, ozoneCfg); } catch (IOException | NodeNotFoundException | TimeoutException e) { - throw new RuntimeException("Can't initialize TestOzoneHDDS: ", e); + throw new RuntimeException("Can't create MockedSCM instance: ", e); } } @@ -96,9 +100,6 @@ public void init(@Nonnull ContainerBalancerConfiguration balancerConfig, @Nonnul */ private void doMock(@Nonnull ContainerBalancerConfiguration cfg, @Nonnull OzoneConfiguration ozoneCfg) throws IOException, NodeNotFoundException, TimeoutException { - containerManager = mockContainerManager(cluster); - mockedReplicaManager = MockedReplicationManager.doMock(); - moveManager = mockMoveManager(); StatefulServiceStateManager stateManager = MockedServiceStateManager.doMock(); SCMServiceManager scmServiceManager = mockSCMServiceManger(); @@ -137,7 +138,7 @@ public String toString() { } public @Nonnull ContainerBalancerTask startBalancerTask(@Nonnull ContainerBalancerConfiguration config) { - init(config); + init(config, new OzoneConfiguration()); return startBalancerTask(new ContainerBalancer(scm), config); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java index 8f1db615dfe3..35804795cc52 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerDatanodeNodeLimit.java @@ -20,6 +20,10 @@ import jakarta.annotation.Nonnull; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.node.DatanodeUsageInfo; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.BeforeAll; @@ -28,13 +32,26 @@ import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.event.Level; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.stream.Stream; +import static org.apache.hadoop.hdds.scm.container.balancer.TestableCluster.RANDOM; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Tests for {@link ContainerBalancerTask} moved from {@link TestContainerBalancerTask} to run them on clusters @@ -170,8 +187,152 @@ public void balancerShouldObeyMaxSizeLeavingSourceLimit(@Nonnull MockedSCM mocke assertNotEquals(0, task.getSizeScheduledForMoveInLatestIteration()); } + /** + * Checks whether ContainerBalancer is correctly updating the list of + * unBalanced nodes with varying values of Threshold. + */ + @ParameterizedTest(name = "MockedSCM #{index}: {0}") + @MethodSource("createMockedSCMs") + public void initializeIterationShouldUpdateUnBalancedNodesWhenThresholdChanges(@Nonnull MockedSCM mockedSCM) { + ContainerBalancerConfiguration config = new OzoneConfiguration().getObject(ContainerBalancerConfiguration.class); + int nodeCount = mockedSCM.getCluster().getNodeCount(); + if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) { + config.setMaxDatanodesPercentageToInvolvePerIteration(100); + } + config.setThreshold(10); + config.setIterations(1); + config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT); + config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT); + + // check for random threshold values + for (int i = 0; i < 50; i++) { + double randomThreshold = RANDOM.nextDouble() * 100; + List expectedUnBalancedNodes = mockedSCM.getCluster().getUnBalancedNodes(randomThreshold); + config.setThreshold(randomThreshold); + + ContainerBalancerTask task = mockedSCM.startBalancerTask(config); + List unBalancedNodesAccordingToBalancer = getUnBalancedNodes(task); + + assertEquals(expectedUnBalancedNodes.size(), unBalancedNodesAccordingToBalancer.size()); + + for (int j = 0; j < expectedUnBalancedNodes.size(); j++) { + assertEquals(expectedUnBalancedNodes.get(j).getDatanodeDetails(), + unBalancedNodesAccordingToBalancer.get(j).getDatanodeDetails()); + } + } + } + + @ParameterizedTest(name = "MockedSCM #{index}: {0}") + @MethodSource("createMockedSCMs") + public void testCalculationOfUtilization(@Nonnull MockedSCM mockedSCM) { + TestableCluster cluster = mockedSCM.getCluster(); + DatanodeUsageInfo[] nodesInCluster = cluster.getNodesInCluster(); + double[] nodeUtilizations = cluster.getNodeUtilizationList(); + assertEquals(nodesInCluster.length, nodeUtilizations.length); + for (int i = 0; i < nodesInCluster.length; i++) { + assertEquals(nodeUtilizations[i], nodesInCluster[i].calculateUtilization(), 0.0001); + } + + // should be equal to average utilization of the cluster + assertEquals(cluster.getAverageUtilization(), + ContainerBalancerTask.calculateAvgUtilization(Arrays.asList(nodesInCluster)), 0.0001); + } + + @ParameterizedTest(name = "MockedSCM #{index}: {0}") + @MethodSource("createMockedSCMs") + public void testBalancerWithMoveManager(@Nonnull MockedSCM mockedSCM) + throws IOException, NodeNotFoundException, TimeoutException { + ContainerBalancerConfiguration config = new OzoneConfiguration().getObject(ContainerBalancerConfiguration.class); + int nodeCount = mockedSCM.getCluster().getNodeCount(); + if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) { + config.setMaxDatanodesPercentageToInvolvePerIteration(100); + } + config.setThreshold(10); + config.setIterations(1); + config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT); + config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT); + + mockedSCM.disableLegacyReplicationManager(); + mockedSCM.startBalancerTask(config); + + verify(mockedSCM.getMoveManager(), atLeastOnce()) + .move(any(ContainerID.class), + any(DatanodeDetails.class), + any(DatanodeDetails.class)); + + verify(mockedSCM.getReplicationManager(), times(0)) + .move(any(ContainerID.class), any( + DatanodeDetails.class), any(DatanodeDetails.class)); + } + + @ParameterizedTest(name = "MockedSCM #{index}: {0}") + @MethodSource("createMockedSCMs") + public void unBalancedNodesListShouldBeEmptyWhenClusterIsBalanced(@Nonnull MockedSCM mockedSCM) { + ContainerBalancerConfiguration config = new OzoneConfiguration().getObject(ContainerBalancerConfiguration.class); + int nodeCount = mockedSCM.getCluster().getNodeCount(); + if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) { + config.setMaxDatanodesPercentageToInvolvePerIteration(100); + } + config.setThreshold(10); + config.setIterations(1); + config.setMaxSizeToMovePerIteration(50 * STORAGE_UNIT); + config.setMaxSizeEnteringTarget(50 * STORAGE_UNIT); + config.setThreshold(99.99); + + ContainerBalancerTask task = mockedSCM.startBalancerTask(config); + ContainerBalancerMetrics metrics = task.getMetrics(); + assertEquals(0, getUnBalancedNodes(task).size()); + assertEquals(0, metrics.getNumDatanodesUnbalanced()); + } + + @ParameterizedTest(name = "MockedSCM #{index}: {0}") + @MethodSource("createMockedSCMs") + public void testMetrics(@Nonnull MockedSCM mockedSCM) throws IOException, NodeNotFoundException { + OzoneConfiguration ozoneConfig = new OzoneConfiguration(); + ozoneConfig.set("hdds.datanode.du.refresh.period", "1ms"); + ContainerBalancerConfiguration config = balancerConfigByOzoneConfig(ozoneConfig); + int nodeCount = mockedSCM.getCluster().getNodeCount(); + if (nodeCount < DATANODE_COUNT_LIMIT_FOR_SMALL_CLUSTER) { + config.setMaxDatanodesPercentageToInvolvePerIteration(100); + } + config.setBalancingInterval(Duration.ofMillis(2)); + config.setThreshold(10); + config.setIterations(1); + config.setMaxSizeEnteringTarget(6 * STORAGE_UNIT); + // deliberately set max size per iteration to a low value, 6 GB + config.setMaxSizeToMovePerIteration(6 * STORAGE_UNIT); + + when(mockedSCM.getMoveManager().move(any(), any(), any())) + .thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY)) + .thenReturn(CompletableFuture.completedFuture(MoveManager.MoveResult.COMPLETED)); + ContainerBalancerTask task = mockedSCM.startBalancerTask(config); + + ContainerBalancerMetrics metrics = task.getMetrics(); + assertEquals(mockedSCM.getCluster().getUnBalancedNodes(config.getThreshold()).size(), + metrics.getNumDatanodesUnbalanced()); + assertThat(metrics.getDataSizeMovedGBInLatestIteration()).isLessThanOrEqualTo(6); + assertThat(metrics.getDataSizeMovedGB()).isGreaterThan(0); + assertEquals(1, metrics.getNumIterations()); + assertThat(metrics.getNumContainerMovesScheduledInLatestIteration()).isGreaterThan(0); + assertEquals(metrics.getNumContainerMovesScheduled(), metrics.getNumContainerMovesScheduledInLatestIteration()); + assertEquals(metrics.getNumContainerMovesScheduled(), + metrics.getNumContainerMovesCompleted() + + metrics.getNumContainerMovesFailed() + + metrics.getNumContainerMovesTimeout()); + assertEquals(0, metrics.getNumContainerMovesTimeout()); + assertEquals(1, metrics.getNumContainerMovesFailed()); + } + + + public static List getUnBalancedNodes(@Nonnull ContainerBalancerTask task) { + ArrayList result = new ArrayList<>(); + result.addAll(task.getOverUtilizedNodes()); + result.addAll(task.getUnderUtilizedNodes()); + return result; + } + private static boolean stillHaveUnbalancedNodes(@Nonnull ContainerBalancerTask task) { - return !task.getUnBalancedNodes().isEmpty(); + return !getUnBalancedNodes(task).isEmpty(); } public static @Nonnull MockedSCM getMockedSCM(int datanodeCount) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java index 3518e4523708..0f4551b45c24 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerTask.java @@ -88,7 +88,6 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; @@ -115,7 +114,6 @@ public class TestContainerBalancerTask { private ContainerBalancerConfiguration balancerConfiguration; private List nodesInCluster; private List nodeUtilizations; - private double averageUtilization; private int numberOfNodes; private Map> cidToReplicasMap = new HashMap<>(); @@ -168,7 +166,7 @@ public void setup(TestInfo testInfo) throws IOException, NodeNotFoundException, .filter(method -> method.getName().equals("balancerShouldMoveOnlyPositiveSizeContainers")) .map(method -> new int[]{0, 0, 0, 0, 0, 1, 2, 3, 4, 5}) .orElse(null); - averageUtilization = createCluster(sizeArray); + createCluster(sizeArray); mockNodeManager = new MockNodeManager(datanodeToContainersMap); NetworkTopology clusterMap = mockNodeManager.getClusterNetworkTopologyMap(); @@ -250,88 +248,6 @@ public void setup(TestInfo testInfo) throws IOException, NodeNotFoundException, sb.getMetrics(), balancerConfiguration, false); } - @Test - public void testCalculationOfUtilization() { - assertEquals(nodesInCluster.size(), nodeUtilizations.size()); - for (int i = 0; i < nodesInCluster.size(); i++) { - assertEquals(nodeUtilizations.get(i), - nodesInCluster.get(i).calculateUtilization(), 0.0001); - } - - // should be equal to average utilization of the cluster - assertEquals(averageUtilization, containerBalancerTask.calculateAvgUtilization(nodesInCluster), 0.0001); - } - - /** - * Checks whether ContainerBalancer is correctly updating the list of - * unBalanced nodes with varying values of Threshold. - */ - @Test - public void - initializeIterationShouldUpdateUnBalancedNodesWhenThresholdChanges() { - List expectedUnBalancedNodes; - List unBalancedNodesAccordingToBalancer; - - // check for random threshold values - ContainerBalancer sb = new ContainerBalancer(scm); - for (int i = 0; i < 50; i++) { - double randomThreshold = RANDOM.nextDouble() * 100; - - expectedUnBalancedNodes = - determineExpectedUnBalancedNodes(randomThreshold); - - balancerConfiguration.setThreshold(randomThreshold); - containerBalancerTask = new ContainerBalancerTask(scm, 0, sb, - sb.getMetrics(), balancerConfiguration, false); - containerBalancerTask.run(); - - unBalancedNodesAccordingToBalancer = - containerBalancerTask.getUnBalancedNodes(); - - assertEquals(expectedUnBalancedNodes.size(), unBalancedNodesAccordingToBalancer.size()); - - for (int j = 0; j < expectedUnBalancedNodes.size(); j++) { - assertEquals(expectedUnBalancedNodes.get(j).getDatanodeDetails(), - unBalancedNodesAccordingToBalancer.get(j).getDatanodeDetails()); - } - } - } - - @Test - public void testBalancerWithMoveManager() - throws IllegalContainerBalancerStateException, IOException, - InvalidContainerBalancerConfigurationException, TimeoutException, - NodeNotFoundException { - rmConf.setEnableLegacy(false); - startBalancer(balancerConfiguration); - verify(moveManager, atLeastOnce()) - .move(any(ContainerID.class), - any(DatanodeDetails.class), - any(DatanodeDetails.class)); - - verify(replicationManager, times(0)) - .move(any(ContainerID.class), any( - DatanodeDetails.class), any(DatanodeDetails.class)); - } - - /** - * Checks whether the list of unBalanced nodes is empty when the cluster is - * balanced. - */ - @Test - public void unBalancedNodesListShouldBeEmptyWhenClusterIsBalanced() - throws IllegalContainerBalancerStateException, IOException, - InvalidContainerBalancerConfigurationException, TimeoutException { - balancerConfiguration.setThreshold(99.99); - startBalancer(balancerConfiguration); - - - stopBalancer(); - ContainerBalancerMetrics metrics = containerBalancerTask.getMetrics(); - assertEquals(0, containerBalancerTask.getUnBalancedNodes().size()); - assertEquals(0, metrics.getNumDatanodesUnbalanced()); - } - @Test public void containerBalancerShouldSelectOnlyClosedContainers() throws IllegalContainerBalancerStateException, IOException, @@ -345,8 +261,7 @@ public void containerBalancerShouldSelectOnlyClosedContainers() stopBalancer(); // balancer should have identified unbalanced nodes - assertFalse(containerBalancerTask.getUnBalancedNodes() - .isEmpty()); + assertFalse(TestContainerBalancerDatanodeNodeLimit.getUnBalancedNodes(containerBalancerTask).isEmpty()); // no container should have been selected assertTrue(containerBalancerTask.getContainerToSourceMap() .isEmpty()); @@ -407,8 +322,7 @@ public void balancerShouldNotSelectNonClosedContainerReplicas() stopBalancer(); // balancer should have identified unbalanced nodes - assertFalse(containerBalancerTask.getUnBalancedNodes() - .isEmpty()); + assertFalse(TestContainerBalancerDatanodeNodeLimit.getUnBalancedNodes(containerBalancerTask).isEmpty()); // no container should have moved because all replicas are CLOSING assertTrue( containerBalancerTask.getContainerToSourceMap().isEmpty()); @@ -587,46 +501,6 @@ public void balancerShouldNotSelectConfiguredExcludeContainers() } } - @Test - public void testMetrics() - throws IllegalContainerBalancerStateException, IOException, - InvalidContainerBalancerConfigurationException, TimeoutException, - NodeNotFoundException { - conf.set("hdds.datanode.du.refresh.period", "1ms"); - balancerConfiguration.setBalancingInterval(Duration.ofMillis(2)); - balancerConfiguration.setThreshold(10); - balancerConfiguration.setIterations(1); - balancerConfiguration.setMaxSizeEnteringTarget(6 * STORAGE_UNIT); - // deliberately set max size per iteration to a low value, 6 GB - balancerConfiguration.setMaxSizeToMovePerIteration(6 * STORAGE_UNIT); - balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); - when(moveManager.move(any(), any(), any())) - .thenReturn(CompletableFuture.completedFuture( - MoveManager.MoveResult.REPLICATION_FAIL_NODE_UNHEALTHY)) - .thenReturn(CompletableFuture.completedFuture( - MoveManager.MoveResult.COMPLETED)); - - startBalancer(balancerConfiguration); - stopBalancer(); - - ContainerBalancerMetrics metrics = containerBalancerTask.getMetrics(); - assertEquals(determineExpectedUnBalancedNodes( - balancerConfiguration.getThreshold()).size(), - metrics.getNumDatanodesUnbalanced()); - assertThat(metrics.getDataSizeMovedGBInLatestIteration()).isLessThanOrEqualTo(6); - assertThat(metrics.getDataSizeMovedGB()).isGreaterThan(0); - assertEquals(1, metrics.getNumIterations()); - assertThat(metrics.getNumContainerMovesScheduledInLatestIteration()).isGreaterThan(0); - assertEquals(metrics.getNumContainerMovesScheduled(), - metrics.getNumContainerMovesScheduledInLatestIteration()); - assertEquals(metrics.getNumContainerMovesScheduled(), - metrics.getNumContainerMovesCompleted() + - metrics.getNumContainerMovesFailed() + - metrics.getNumContainerMovesTimeout()); - assertEquals(0, metrics.getNumContainerMovesTimeout()); - assertEquals(1, metrics.getNumContainerMovesFailed()); - } - /** * Tests if {@link ContainerBalancer} follows the includeNodes and * excludeNodes configurations in {@link ContainerBalancerConfiguration}. @@ -1077,35 +951,6 @@ public void balancerShouldMoveOnlyPositiveSizeContainers() assertFalse(zeroOrNegSizeContainerMoved); } - /** - * Determines unBalanced nodes, that is, over and under utilized nodes, - * according to the generated utilization values for nodes and the threshold. - * - * @param threshold A percentage in the range 0 to 100 - * @return List of DatanodeUsageInfo containing the expected(correct) - * unBalanced nodes. - */ - private List determineExpectedUnBalancedNodes( - double threshold) { - threshold /= 100; - double lowerLimit = averageUtilization - threshold; - double upperLimit = averageUtilization + threshold; - - // use node utilizations to determine over and under utilized nodes - List expectedUnBalancedNodes = new ArrayList<>(); - for (int i = 0; i < numberOfNodes; i++) { - if (nodeUtilizations.get(numberOfNodes - i - 1) > upperLimit) { - expectedUnBalancedNodes.add(nodesInCluster.get(numberOfNodes - i - 1)); - } - } - for (int i = 0; i < numberOfNodes; i++) { - if (nodeUtilizations.get(i) < lowerLimit) { - expectedUnBalancedNodes.add(nodesInCluster.get(i)); - } - } - return expectedUnBalancedNodes; - } - /** * Generates a range of equally spaced utilization(that is, used / capacity) * values from 0 to 1. @@ -1132,10 +977,9 @@ private void generateUtilizations(int count) throws IllegalArgumentException { * cluster have utilization values determined by generateUtilizations method. * @return average utilization (used space / capacity) of the cluster */ - private double createCluster(int[] sizeArray) { + private void createCluster(int[] sizeArray) { generateData(sizeArray); createReplicasForContainers(); - long clusterCapacity = 0, clusterUsedSpace = 0; // for each node utilization, calculate that datanode's used space and // capacity @@ -1158,10 +1002,7 @@ private double createCluster(int[] sizeArray) { datanodeCapacity - datanodeUsedSpace, 0, datanodeCapacity - datanodeUsedSpace - 1); nodesInCluster.get(i).setScmNodeStat(stat); - clusterUsedSpace += datanodeUsedSpace; - clusterCapacity += datanodeCapacity; } - return (double) clusterUsedSpace / clusterCapacity; } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestableCluster.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestableCluster.java index 60ee45535f0b..a565fac0e3d6 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestableCluster.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestableCluster.java @@ -138,8 +138,12 @@ int getNodeCount() { // Use node utilization to determine over and under utilized nodes. List expectedUnBalancedNodes = new ArrayList<>(); for (int i = 0; i < nodeCount; i++) { - double nodeUtilization = nodeUtilizationList[i]; - if (nodeUtilization < lowerLimit || nodeUtilization > upperLimit) { + if (nodeUtilizationList[nodeCount - i - 1] > upperLimit) { + expectedUnBalancedNodes.add(nodesInCluster[nodeCount - i - 1]); + } + } + for (int i = 0; i < nodeCount; i++) { + if (nodeUtilizationList[i] < lowerLimit) { expectedUnBalancedNodes.add(nodesInCluster[i]); } }