From 6170d80fab2ddc0fd94090fcf0a94453b16ef9d1 Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Tue, 7 Dec 2021 12:11:59 +0530 Subject: [PATCH 1/6] HDDS-5927. Improve defaults in ContainerBalancerConfiguration --- .../hadoop/hdds/scm/client/ScmClient.java | 2 +- .../StorageContainerLocationProtocol.java | 2 +- ...ocationProtocolClientSideTranslatorPB.java | 12 ++--- .../src/main/proto/ScmAdminProtocol.proto | 2 +- .../src/main/resources/proto.lock | 2 +- .../container/balancer/ContainerBalancer.java | 12 ++--- .../ContainerBalancerConfiguration.java | 49 ++++++++++--------- ...ocationProtocolServerSideTranslatorPB.java | 8 +-- .../scm/server/SCMClientProtocolServer.java | 12 ++--- .../balancer/TestContainerBalancer.java | 10 ++-- .../scm/cli/ContainerBalancerCommands.java | 4 +- .../cli/ContainerBalancerStartSubcommand.java | 9 ++-- .../scm/cli/ContainerOperationClient.java | 4 +- .../TestContainerBalancerOperations.java | 6 +-- 14 files changed, 68 insertions(+), 66 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 351870a3cd15..25833cb0632f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -312,7 +312,7 @@ Map> getSafeModeRuleStatuses() * Start ContainerBalancer. */ boolean startContainerBalancer(Optional threshold, - Optional idleiterations, + Optional iterations, Optional maxDatanodesRatioToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 7f8663ef05b0..61f6ca7136f0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -312,7 +312,7 @@ Map> getSafeModeRuleStatuses() * Start ContainerBalancer. */ boolean startContainerBalancer(Optional threshold, - Optional idleiterations, + Optional iterations, Optional maxDatanodesRatioToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index c484420ba047..c4de8e0ac1da 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -737,7 +737,7 @@ public boolean getReplicationManagerStatus() throws IOException { @Override public boolean startContainerBalancer( - Optional threshold, Optional idleiterations, + Optional threshold, Optional iterations, Optional maxDatanodesRatioToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, @@ -769,12 +769,12 @@ public boolean startContainerBalancer( "lesser than equal to one."); builder.setMaxDatanodesRatioToInvolvePerIteration(mdti); } - if (idleiterations.isPresent()) { - int idi = idleiterations.get(); - Preconditions.checkState(idi > 0 || idi == -1, - "idleiterations must be positive or" + + if (iterations.isPresent()) { + int i = iterations.get(); + Preconditions.checkState(i > 0 || i == -1, + "iterations must be positive or" + " -1(infinitly run container balancer)."); - builder.setIdleiterations(idi); + builder.setIterations(i); } if (maxSizeEnteringTargetInGB.isPresent()) { diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto index 4830c11b9be4..f423c42e6471 100644 --- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto +++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto @@ -479,7 +479,7 @@ message GetContainerTokenResponseProto { message StartContainerBalancerRequestProto { optional string traceID = 1; optional double threshold = 2; - optional int32 idleiterations = 3; + optional int32 iterations = 3; optional double maxDatanodesRatioToInvolvePerIteration = 4; optional int64 maxSizeToMovePerIterationInGB = 5; optional int64 maxSizeEnteringTargetInGB = 6; diff --git a/hadoop-hdds/interface-admin/src/main/resources/proto.lock b/hadoop-hdds/interface-admin/src/main/resources/proto.lock index ec40a30649d4..09539c0c3219 100644 --- a/hadoop-hdds/interface-admin/src/main/resources/proto.lock +++ b/hadoop-hdds/interface-admin/src/main/resources/proto.lock @@ -1334,7 +1334,7 @@ }, { "id": 3, - "name": "idleiterations", + "name": "iterations", "type": "int32" }, { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 91dd1b240807..425f4c787a14 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -72,7 +72,7 @@ public class ContainerBalancer { private long maxSizeToMovePerIteration; private int countDatanodesInvolvedPerIteration; private long sizeMovedPerIteration; - private int idleIteration; + private int iterations; private List unBalancedNodes; private List overUtilizedNodes; private List underUtilizedNodes; @@ -170,16 +170,16 @@ public boolean start(ContainerBalancerConfiguration balancerConfiguration) { * Balances the cluster. */ private void balance() { - this.idleIteration = config.getIdleIteration(); - if(this.idleIteration == -1) { + this.iterations = config.getIterations(); + if(this.iterations == -1) { //run balancer infinitely - this.idleIteration = Integer.MAX_VALUE; + this.iterations = Integer.MAX_VALUE; } this.threshold = config.getThreshold(); this.maxDatanodesRatioToInvolvePerIteration = config.getMaxDatanodesRatioToInvolvePerIteration(); this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration(); - for (int i = 0; i < idleIteration && balancerRunning; i++) { + for (int i = 0; i < iterations && balancerRunning; i++) { // stop balancing if iteration is not initialized if (!initializeIteration()) { stop(); @@ -202,7 +202,7 @@ private void balance() { // wait for configured time before starting next iteration, unless // this was the final iteration - if (i != idleIteration - 1) { + if (i != iterations - 1) { synchronized (this) { try { wait(config.getBalancingInterval().toMillis()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index a0199e0c748f..e16e22974dc2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -76,22 +76,23 @@ public final class ContainerBalancerConfiguration { defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can enter a target datanode in each " + "iteration while balancing. This is the sum of data from multiple " + - "sources. The default value is greater than the configured" + - " (or default) ozone.scm.container.size by 1GB.") + "sources. By default, five times the configured (or default) ozone.scm" + + ".container.size is allowed.") private long maxSizeEnteringTarget; @Config(key = "size.leaving.source.max", type = ConfigType.SIZE, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can leave a source datanode in each " + "iteration while balancing. This is the sum of data moving to multiple " + - "targets. The default value is greater than the configured" + - " (or default) ozone.scm.container.size by 1GB.") + "targets. By default, five times the configured (or default) ozone.scm" + + ".container.size is allowed.") private long maxSizeLeavingSource; - @Config(key = "idle.iterations", type = ConfigType.INT, + @Config(key = "iterations", type = ConfigType.INT, defaultValue = "10", tags = {ConfigTag.BALANCER}, - description = "The idle iteration count of Container Balancer.") - private int idleIterations = 10; + description = "The number of iterations that Container Balancer will " + + "run for.") + private int iterations = 10; @Config(key = "exclude.containers", type = ConfigType.STRING, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "List of container IDs " + @@ -136,14 +137,13 @@ public ContainerBalancerConfiguration(OzoneConfiguration config) { "OzoneConfiguration should not be null."); this.ozoneConfiguration = config; - // maxSizeEnteringTarget and maxSizeLeavingSource should by default be - // greater than container size - long size = (long) ozoneConfiguration.getStorageSize( + // maxSizeEnteringTarget and maxSizeLeavingSource should by default allow + // five containers + long size = 5 * (long) ozoneConfiguration.getStorageSize( ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB) + - OzoneConsts.GB; - maxSizeEnteringTarget = size; - maxSizeLeavingSource = size; + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + maxSizeEnteringTarget = size + OzoneConsts.GB; + maxSizeLeavingSource = size + OzoneConsts.GB; // balancing interval should be greater than DUFactory refresh period duConf = ozoneConfiguration.getObject(DUFactory.Conf.class); @@ -174,26 +174,27 @@ public void setThreshold(double threshold) { } /** - * Gets the idle iteration value for Container Balancer. + * Gets the iteration count for Container Balancer. A value of -1 means + * infinite number of iterations. * - * @return a idle iteration count larger than 0 + * @return a value greater than 0, or -1 */ - public int getIdleIteration() { - return idleIterations; + public int getIterations() { + return iterations; } /** - * Sets the idle iteration value for Container Balancer. + * Sets the number of iterations for Container Balancer. * - * @param count a idle iteration count larger than 0 + * @param count a value greater than 0, or -1 */ - public void setIdleIteration(int count) { + public void setIterations(int count) { if (count < -1 || 0 == count) { throw new IllegalArgumentException( - "Idle iteration count must be larger than 0 or " + - "-1(for infinitely running)."); + "Iteration count must be greater than 0, or " + + "-1(for running balancer infinitely)."); } - this.idleIterations = count; + this.iterations = count; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 3a471fbb0b06..6bc2ab6b0ee4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -710,7 +710,7 @@ public StartContainerBalancerResponseProto startContainerBalancer( StartContainerBalancerRequestProto request) throws IOException { Optional threshold = Optional.empty(); - Optional idleiterations = Optional.empty(); + Optional iterations = Optional.empty(); Optional maxDatanodesRatioToInvolvePerIteration = Optional.empty(); Optional maxSizeToMovePerIterationInGB = Optional.empty(); Optional maxSizeEnteringTargetInGB = Optional.empty(); @@ -719,8 +719,8 @@ public StartContainerBalancerResponseProto startContainerBalancer( if(request.hasThreshold()) { threshold = Optional.of(request.getThreshold()); } - if(request.hasIdleiterations()) { - idleiterations = Optional.of(request.getIdleiterations()); + if(request.hasIterations()) { + iterations = Optional.of(request.getIterations()); } if(request.hasMaxDatanodesRatioToInvolvePerIteration()) { maxDatanodesRatioToInvolvePerIteration = @@ -743,7 +743,7 @@ public StartContainerBalancerResponseProto startContainerBalancer( return StartContainerBalancerResponseProto.newBuilder(). setStart(impl.startContainerBalancer(threshold, - idleiterations, maxDatanodesRatioToInvolvePerIteration, + iterations, maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB)).build(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 6fbd8c6ee657..b1408e94006e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -744,7 +744,7 @@ public StatusAndMessages queryUpgradeFinalizationProgress( @Override public boolean startContainerBalancer( - Optional threshold, Optional idleiterations, + Optional threshold, Optional iterations, Optional maxDatanodesRatioToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTarget, @@ -774,12 +774,12 @@ public boolean startContainerBalancer( "lesser than or equal to one."); cbc.setMaxDatanodesRatioToInvolvePerIteration(mdti); } - if (idleiterations.isPresent()) { - int idi = idleiterations.get(); - Preconditions.checkState(idi > 0 || idi == -1, - "idleiterations must be positive or" + + if (iterations.isPresent()) { + int i = iterations.get(); + Preconditions.checkState(i > 0 || i == -1, + "iterations must be positive or" + " -1(infinitly run container balancer)."); - cbc.setIdleIteration(idi); + cbc.setIterations(i); } if (maxSizeEnteringTarget.isPresent()) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java index e260bb3487d1..4d74c9af79cb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java @@ -104,7 +104,7 @@ public void setup() throws SCMException, NodeNotFoundException { balancerConfiguration = new ContainerBalancerConfiguration(conf); balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setIdleIteration(1); + balancerConfiguration.setIterations(1); balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); @@ -231,7 +231,7 @@ public void containerBalancerShouldObeyMaxDatanodesToInvolveLimit() { balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(0.3d); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); balancerConfiguration.setThreshold(0.01); - balancerConfiguration.setIdleIteration(1); + balancerConfiguration.setIterations(1); containerBalancer.start(balancerConfiguration); // waiting for balance completed. @@ -297,7 +297,7 @@ public void containerBalancerShouldSelectOnlyClosedContainers() { public void containerBalancerShouldObeyMaxSizeToMoveLimit() { balancerConfiguration.setThreshold(0.01); balancerConfiguration.setMaxSizeToMovePerIteration(10 * OzoneConsts.GB); - balancerConfiguration.setIdleIteration(1); + balancerConfiguration.setIterations(1); containerBalancer.start(balancerConfiguration); // waiting for balance completed. @@ -505,7 +505,7 @@ public void balancerShouldObeyMaxSizeEnteringTargetLimit() { @Test public void testMetrics() { balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setIdleIteration(1); + balancerConfiguration.setIterations(1); balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0); @@ -540,7 +540,7 @@ public void testMetrics() { @Test public void balancerShouldFollowExcludeAndIncludeDatanodesConfigurations() { balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setIdleIteration(1); + balancerConfiguration.setIterations(1); balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java index 304165160689..3bea380b1882 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java @@ -41,7 +41,7 @@ * To start: * ozone admin containerbalancer start * [ -t/--threshold {@literal }] - * [ -i/--idleiterations {@literal }] + * [ -i/--iterations {@literal }] * [ -d/--maxDatanodesRatioToInvolvePerIteration * {@literal }] * [ -s/--maxSizeToMovePerIterationInGB @@ -52,7 +52,7 @@ * ozone admin containerbalancer start -t 0.05 * start balancer with a threshold of 5% * ozone admin containerbalancer start -i 20 - * start balancer with maximum 20 consecutive idle iterations + * start balancer with maximum 20 consecutive iterations * ozone admin containerbalancer start -i 0 * run balancer infinitely with default values in the configuration * ozone admin containerbalancer start -d 0.4 diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index a4ee0d2c5544..a8820d5c7c6c 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -39,9 +39,10 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { description = "Threshold target whether the cluster is balanced") private Optional threshold; - @Option(names = {"-i", "--idleiterations"}, - description = "Maximum consecutive idle iterations") - private Optional idleiterations; + @Option(names = {"-i", "--iterations"}, + description = "Maximum consecutive iterations that" + + " balancer will run for") + private Optional iterations; @Option(names = {"-d", "--maxDatanodesRatioToInvolvePerIteration"}, description = "The ratio of maximum number of datanodes that should be " + @@ -66,7 +67,7 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { @Override public void execute(ScmClient scmClient) throws IOException { - boolean result = scmClient.startContainerBalancer(threshold, idleiterations, + boolean result = scmClient.startContainerBalancer(threshold, iterations, maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); if (result) { diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 30f0ec8d46d2..182c77e39ab0 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -552,14 +552,14 @@ public boolean getReplicationManagerStatus() throws IOException { @Override public boolean startContainerBalancer( - Optional threshold, Optional idleiterations, + Optional threshold, Optional iterations, Optional maxDatanodesRatioToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, Optional maxSizeLeavingSourceInGB) throws IOException { return storageContainerLocationClient.startContainerBalancer(threshold, - idleiterations, maxDatanodesRatioToInvolvePerIteration, + iterations, maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java index ac5b737ab3ea..9a84453cd0b6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java @@ -81,13 +81,13 @@ public void testContainerBalancerCLIOperations() throws Exception { boolean running = containerBalancerClient.getContainerBalancerStatus(); assertFalse(running); Optional threshold = Optional.of(0.1); - Optional idleiterations = Optional.of(10000); + Optional iterations = Optional.of(10000); Optional maxDatanodesRatioToInvolvePerIteration = Optional.of(1d); Optional maxSizeToMovePerIterationInGB = Optional.of(1L); Optional maxSizeEnteringTargetInGB = Optional.of(1L); Optional maxSizeLeavingSourceInGB = Optional.of(1L); - containerBalancerClient.startContainerBalancer(threshold, idleiterations, + containerBalancerClient.startContainerBalancer(threshold, iterations, maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); running = containerBalancerClient.getContainerBalancerStatus(); @@ -104,7 +104,7 @@ public void testContainerBalancerCLIOperations() throws Exception { assertFalse(running); // test normally start , and stop it before balance is completed - containerBalancerClient.startContainerBalancer(threshold, idleiterations, + containerBalancerClient.startContainerBalancer(threshold, iterations, maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); running = containerBalancerClient.getContainerBalancerStatus(); From 36ddd2e8894f4966bda87059d39cd8b3f37f6af7 Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Wed, 22 Dec 2021 11:46:03 +0530 Subject: [PATCH 2/6] refactor configurations while maintaining backwards compatibility --- .../hadoop/hdds/scm/client/ScmClient.java | 2 +- .../StorageContainerLocationProtocol.java | 2 +- ...ocationProtocolClientSideTranslatorPB.java | 20 +-- .../src/main/proto/ScmAdminProtocol.proto | 7 +- .../src/main/resources/proto.lock | 2 +- .../container/balancer/ContainerBalancer.java | 40 ++++- .../ContainerBalancerConfiguration.java | 144 +++++++----------- ...ocationProtocolServerSideTranslatorPB.java | 23 ++- .../scm/server/SCMClientProtocolServer.java | 26 ++-- .../balancer/TestContainerBalancer.java | 51 +++++-- .../scm/cli/ContainerBalancerCommands.java | 15 +- .../cli/ContainerBalancerStartSubcommand.java | 15 +- .../scm/cli/ContainerOperationClient.java | 4 +- .../TestContainerBalancerOperations.java | 13 +- 14 files changed, 193 insertions(+), 171 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 25833cb0632f..342a3e7bd8d9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -313,7 +313,7 @@ Map> getSafeModeRuleStatuses() */ boolean startContainerBalancer(Optional threshold, Optional iterations, - Optional maxDatanodesRatioToInvolvePerIteration, + Optional maxDatanodesPercentageToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, Optional maxSizeLeavingSourceInGB) throws IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 61f6ca7136f0..b688af3ab733 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -313,7 +313,7 @@ Map> getSafeModeRuleStatuses() */ boolean startContainerBalancer(Optional threshold, Optional iterations, - Optional maxDatanodesRatioToInvolvePerIteration, + Optional maxDatanodesPercentageToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, Optional maxSizeLeavingSourceInGB) throws IOException; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index c4de8e0ac1da..2e6cddcb7e6f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -738,7 +738,7 @@ public boolean getReplicationManagerStatus() throws IOException { @Override public boolean startContainerBalancer( Optional threshold, Optional iterations, - Optional maxDatanodesRatioToInvolvePerIteration, + Optional maxDatanodesPercentageToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, Optional maxSizeLeavingSourceInGB) throws IOException{ @@ -759,21 +759,21 @@ public boolean startContainerBalancer( "maxSizeToMovePerIterationInGB must be positive."); builder.setMaxSizeToMovePerIterationInGB(mstm); } - if (maxDatanodesRatioToInvolvePerIteration.isPresent()) { - double mdti = maxDatanodesRatioToInvolvePerIteration.get(); + if (maxDatanodesPercentageToInvolvePerIteration.isPresent()) { + int mdti = maxDatanodesPercentageToInvolvePerIteration.get(); Preconditions.checkState(mdti >= 0, - "maxDatanodesRatioToInvolvePerIteration must be " + + "maxDatanodesPercentageToInvolvePerIteration must be " + "greater than equal to zero."); - Preconditions.checkState(mdti <= 1, - "maxDatanodesRatioToInvolvePerIteration must be " + - "lesser than equal to one."); - builder.setMaxDatanodesRatioToInvolvePerIteration(mdti); + Preconditions.checkState(mdti <= 100, + "maxDatanodesPercentageToInvolvePerIteration must be " + + "lesser than equal to hundred."); + builder.setMaxDatanodesPercentageToInvolvePerIteration(mdti); } if (iterations.isPresent()) { int i = iterations.get(); Preconditions.checkState(i > 0 || i == -1, - "iterations must be positive or" + - " -1(infinitly run container balancer)."); + "number of iterations must be positive or" + + " -1 (for running container balancer infinitely)."); builder.setIterations(i); } diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto index f423c42e6471..980dfa9f2ec1 100644 --- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto +++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto @@ -479,11 +479,14 @@ message GetContainerTokenResponseProto { message StartContainerBalancerRequestProto { optional string traceID = 1; optional double threshold = 2; - optional int32 iterations = 3; - optional double maxDatanodesRatioToInvolvePerIteration = 4; + optional int32 idleiterations = 3 [deprecated = true]; + optional double maxDatanodesRatioToInvolvePerIteration = 4 [deprecated = + true]; optional int64 maxSizeToMovePerIterationInGB = 5; optional int64 maxSizeEnteringTargetInGB = 6; optional int64 maxSizeLeavingSourceInGB = 7; + optional int32 maxDatanodesPercentageToInvolvePerIteration = 8; + optional int32 iterations = 9; } message StartContainerBalancerResponseProto { diff --git a/hadoop-hdds/interface-admin/src/main/resources/proto.lock b/hadoop-hdds/interface-admin/src/main/resources/proto.lock index 09539c0c3219..ec40a30649d4 100644 --- a/hadoop-hdds/interface-admin/src/main/resources/proto.lock +++ b/hadoop-hdds/interface-admin/src/main/resources/proto.lock @@ -1334,7 +1334,7 @@ }, { "id": 3, - "name": "iterations", + "name": "idleiterations", "type": "int32" }, { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 425f4c787a14..3199a5eb5c20 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.fs.DUFactory; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -120,7 +121,6 @@ public ContainerBalancer( this.containerManager = containerManager; this.replicationManager = replicationManager; this.ozoneConfiguration = ozoneConfiguration; - this.config = new ContainerBalancerConfiguration(ozoneConfiguration); this.metrics = ContainerBalancerMetrics.create(); this.scmContext = scmContext; @@ -151,7 +151,8 @@ public boolean start(ContainerBalancerConfiguration balancerConfiguration) { balancerRunning = true; this.config = balancerConfiguration; - this.ozoneConfiguration = config.getOzoneConfiguration(); + ozoneConfiguration.setFromObject(balancerConfiguration); + validateConfiguration(config); LOG.info("Starting Container Balancer...{}", this); //we should start a new balancer thread async @@ -175,10 +176,6 @@ private void balance() { //run balancer infinitely this.iterations = Integer.MAX_VALUE; } - this.threshold = config.getThreshold(); - this.maxDatanodesRatioToInvolvePerIteration = - config.getMaxDatanodesRatioToInvolvePerIteration(); - this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration(); for (int i = 0; i < iterations && balancerRunning; i++) { // stop balancing if iteration is not initialized if (!initializeIteration()) { @@ -244,6 +241,10 @@ private boolean initializeIteration() { } return false; } + this.threshold = config.getThreshold(); + this.maxDatanodesRatioToInvolvePerIteration = + config.getMaxDatanodesRatioToInvolvePerIteration(); + this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration(); this.excludeNodes = config.getExcludeNodes(); this.includeNodes = config.getIncludeNodes(); // include/exclude nodes from balancing according to configs @@ -506,7 +507,7 @@ private ContainerMoveSelection matchSourceWithTarget(DatanodeDetails source) { } /** - * Checks if limits maxDatanodesRatioToInvolvePerIteration and + * Checks if limits maxDatanodesPercentageToInvolvePerIteration and * maxSizeToMovePerIteration have not been hit. * * @return {@link IterationResult#MAX_DATANODES_TO_INVOLVE_REACHED} if reached @@ -752,6 +753,31 @@ public void stop() { LOG.info("Container Balancer stopped successfully."); } + private void validateConfiguration(ContainerBalancerConfiguration conf) { + // maxSizeEnteringTarget and maxSizeLeavingSource should by default be + // greater than container size + long size = (long) ozoneConfiguration.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + + if (conf.getMaxSizeEnteringTarget() <= size) { + LOG.info("MaxSizeEnteringTarget should be larger than " + + "ozone.scm.container.size"); + } + if (conf.getMaxSizeLeavingSource() <= size) { + LOG.info("MaxSizeLeavingSource should be larger than " + + "ozone.scm.container.size"); + } + + // balancing interval should be greater than DUFactory refresh period + DUFactory.Conf duConf = ozoneConfiguration.getObject(DUFactory.Conf.class); + long balancingInterval = duConf.getRefreshPeriod().toMillis(); + if (conf.getBalancingInterval().toMillis() <= balancingInterval) { + LOG.info("balancing.iteration.interval should be larger than " + + "hdds.datanode.du.refresh.period."); + } + } + public void setNodeManager(NodeManager nodeManager) { this.nodeManager = nodeManager; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index e16e22974dc2..10da567f31c2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -18,15 +18,10 @@ package org.apache.hadoop.hdds.scm.container.balancer; -import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.conf.Config; import org.apache.hadoop.hdds.conf.ConfigGroup; import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.ConfigType; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.fs.DUFactory; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.ozone.OzoneConsts; import org.slf4j.Logger; @@ -37,7 +32,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -47,7 +41,6 @@ public final class ContainerBalancerConfiguration { private static final Logger LOG = LoggerFactory.getLogger(ContainerBalancerConfiguration.class); - private OzoneConfiguration ozoneConfiguration; @Config(key = "utilization.threshold", type = ConfigType.AUTO, defaultValue = "0.1", tags = {ConfigTag.BALANCER}, @@ -58,34 +51,32 @@ public final class ContainerBalancerConfiguration { " of the entire cluster) no more than the threshold value.") private String threshold = "0.1"; - @Config(key = "datanodes.involved.max.ratio.per.iteration", type = - ConfigType.AUTO, - defaultValue = "0.2", tags = {ConfigTag.BALANCER}, description = "The " + - "ratio of maximum number of datanodes that should be involved in " + - "balancing in one iteration to the total number of healthy, in service " + - "nodes known to container balancer.") - private String maxDatanodesRatioToInvolvePerIteration = "0.2"; + @Config(key = "datanodes.involved.max.percentage.per.iteration", type = + ConfigType.INT, defaultValue = "20", tags = {ConfigTag.BALANCER}, + description = "Maximum percentage of healthy, in service datanodes " + + "that can be involved in balancing in one iteration.") + private int maxDatanodesPercentageToInvolvePerIteration = 20; @Config(key = "size.moved.max.per.iteration", type = ConfigType.SIZE, - defaultValue = "30GB", tags = {ConfigTag.BALANCER}, + defaultValue = "500GB", tags = {ConfigTag.BALANCER}, description = "The maximum size of data in bytes that will be moved " + "by Container Balancer in one iteration.") - private long maxSizeToMovePerIteration = 30 * OzoneConsts.GB; + private long maxSizeToMovePerIteration = 500 * OzoneConsts.GB; @Config(key = "size.entering.target.max", type = ConfigType.SIZE, - defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + + defaultValue = "26GB", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can enter a target datanode in each " + "iteration while balancing. This is the sum of data from multiple " + - "sources. By default, five times the configured (or default) ozone.scm" + - ".container.size is allowed.") + "sources. The value must be greater than the configured" + + " (or default) ozone.scm.container.size.") private long maxSizeEnteringTarget; @Config(key = "size.leaving.source.max", type = ConfigType.SIZE, - defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " + + defaultValue = "26GB", tags = {ConfigTag.BALANCER}, description = "The " + "maximum size that can leave a source datanode in each " + "iteration while balancing. This is the sum of data moving to multiple " + - "targets. By default, five times the configured (or default) ozone.scm" + - ".container.size is allowed.") + "targets. The value must be greater than the configured" + + " (or default) ozone.scm.container.size.") private long maxSizeLeavingSource; @Config(key = "iterations", type = ConfigType.INT, @@ -100,16 +91,16 @@ public final class ContainerBalancerConfiguration { private String excludeContainers = ""; @Config(key = "move.timeout", type = ConfigType.TIME, defaultValue = "30m", - timeUnit = TimeUnit.MINUTES, tags = {ConfigTag.BALANCER}, description = + tags = {ConfigTag.BALANCER}, description = "The amount of time in minutes to allow a single container to move " + "from source to target.") private long moveTimeout = Duration.ofMinutes(30).toMillis(); @Config(key = "balancing.iteration.interval", type = ConfigType.TIME, - defaultValue = "1h", timeUnit = TimeUnit.MINUTES, tags = { + defaultValue = "70m", tags = { ConfigTag.BALANCER}, description = "The interval period between each " + "iteration of Container Balancer.") - private long balancingInterval; + private long balancingInterval = Duration.ofMinutes(70).toMillis(); @Config(key = "include.datanodes", type = ConfigType.STRING, defaultValue = "", tags = {ConfigTag.BALANCER}, description = "A list of Datanode " + @@ -125,32 +116,6 @@ public final class ContainerBalancerConfiguration { " by default.") private String excludeNodes = ""; - private DUFactory.Conf duConf; - - /** - * Create configuration with default values. - * - * @param config Ozone configuration - */ - public ContainerBalancerConfiguration(OzoneConfiguration config) { - Preconditions.checkNotNull(config, - "OzoneConfiguration should not be null."); - this.ozoneConfiguration = config; - - // maxSizeEnteringTarget and maxSizeLeavingSource should by default allow - // five containers - long size = 5 * (long) ozoneConfiguration.getStorageSize( - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, - ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); - maxSizeEnteringTarget = size + OzoneConsts.GB; - maxSizeLeavingSource = size + OzoneConsts.GB; - - // balancing interval should be greater than DUFactory refresh period - duConf = ozoneConfiguration.getObject(DUFactory.Conf.class); - balancingInterval = duConf.getRefreshPeriod().toMillis() + - Duration.ofMinutes(10).toMillis(); - } - /** * Gets the threshold value for Container Balancer. * @@ -186,7 +151,7 @@ public int getIterations() { /** * Sets the number of iterations for Container Balancer. * - * @param count a value greater than 0, or -1 + * @param count a value greater than 0, or -1 for running balancer infinitely */ public void setIterations(int count) { if (count < -1 || 0 == count) { @@ -198,37 +163,46 @@ public void setIterations(int count) { } /** - * Gets the ratio of maximum number of datanodes that will be involved in - * balancing by Container Balancer in one iteration to the total number of - * healthy, in-service nodes known to balancer. + * Gets the maximum percentage of healthy, in-service datanodes that will be + * involved in balancing in one iteration. * - * @return maximum datanodes to involve divided by total healthy, - * in-service nodes + * @return percentage as an integer from 0 up to and including 100 + */ + public int getMaxDatanodesPercentageToInvolvePerIteration() { + return maxDatanodesPercentageToInvolvePerIteration; + } + + /** + * Gets the ratio of maximum datanodes involved in balancing to the total + * number of healthy, in-service datanodes known to SCM. + * + * @return ratio as a double from 0 up to and including 1 */ public double getMaxDatanodesRatioToInvolvePerIteration() { - return Double.parseDouble(maxDatanodesRatioToInvolvePerIteration); + return maxDatanodesPercentageToInvolvePerIteration / 100d; } /** - * Sets the ratio of maximum number of datanodes that will be involved in - * balancing by Container Balancer in one iteration to the total number of - * healthy, in-service nodes known to balancer. + * Sets the maximum percentage of healthy, in-service datanodes that will be + * involved in balancing in one iteration. * - * @param maxDatanodesRatioToInvolvePerIteration number of datanodes to - * involve divided by total - * number of healthy, in - * service nodes + * @param maxDatanodesPercentageToInvolvePerIteration number of datanodes + * to involve divided by + * total number of + * healthy, in-service + * datanodes multiplied + * by 100 */ - public void setMaxDatanodesRatioToInvolvePerIteration( - double maxDatanodesRatioToInvolvePerIteration) { - if (maxDatanodesRatioToInvolvePerIteration < 0 || - maxDatanodesRatioToInvolvePerIteration > 1) { - throw new IllegalArgumentException("Max datanodes to involve ratio must" + - " be a double greater than equal to zero and lesser than equal to " + - "one."); + public void setMaxDatanodesPercentageToInvolvePerIteration( + int maxDatanodesPercentageToInvolvePerIteration) { + if (maxDatanodesPercentageToInvolvePerIteration < 0 || + maxDatanodesPercentageToInvolvePerIteration > 100) { + throw new IllegalArgumentException(String.format("Argument %d is " + + "illegal. Percentage must be from 0 up to and including 100.", + maxDatanodesPercentageToInvolvePerIteration)); } - this.maxDatanodesRatioToInvolvePerIteration = - String.valueOf(maxDatanodesRatioToInvolvePerIteration); + this.maxDatanodesPercentageToInvolvePerIteration = + maxDatanodesPercentageToInvolvePerIteration; } /** @@ -300,12 +274,7 @@ public Duration getBalancingInterval() { } public void setBalancingInterval(Duration balancingInterval) { - if (balancingInterval.toMillis() > duConf.getRefreshPeriod().toMillis()) { - this.balancingInterval = balancingInterval.toMillis(); - } else { - LOG.warn("Balancing interval duration must be greater than du refresh " + - "period, {} milliseconds", duConf.getRefreshPeriod().toMillis()); - } + this.balancingInterval = balancingInterval.toMillis(); } /** @@ -357,26 +326,17 @@ public void setExcludeNodes(String excludeNodes) { this.excludeNodes = excludeNodes; } - /** - * Gets the {@link OzoneConfiguration} using which this configuration was - * constructed. - * @return the {@link OzoneConfiguration} being used by this configuration - */ - public OzoneConfiguration getOzoneConfiguration() { - return this.ozoneConfiguration; - } - @Override public String toString() { return String.format("Container Balancer Configuration values:%n" + "%-50s %s%n" + "%-50s %s%n" + - "%-50s %s%n" + + "%-50s %d%n" + "%-50s %dGB%n"+ "%-50s %dGB%n"+ "%-50s %dGB%n", "Key", "Value", "Threshold", - threshold, "Max Datanodes to Involve per Iteration(ratio)", - maxDatanodesRatioToInvolvePerIteration, + threshold, "Max Datanodes to Involve per Iteration(percent)", + maxDatanodesPercentageToInvolvePerIteration, "Max Size to Move per Iteration", maxSizeToMovePerIteration / OzoneConsts.GB, "Max Size Entering Target per Iteration", diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 6bc2ab6b0ee4..460bc8c5a3ee 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -711,7 +711,8 @@ public StartContainerBalancerResponseProto startContainerBalancer( throws IOException { Optional threshold = Optional.empty(); Optional iterations = Optional.empty(); - Optional maxDatanodesRatioToInvolvePerIteration = Optional.empty(); + Optional maxDatanodesPercentageToInvolvePerIteration = + Optional.empty(); Optional maxSizeToMovePerIterationInGB = Optional.empty(); Optional maxSizeEnteringTargetInGB = Optional.empty(); Optional maxSizeLeavingSourceInGB = Optional.empty(); @@ -719,13 +720,21 @@ public StartContainerBalancerResponseProto startContainerBalancer( if(request.hasThreshold()) { threshold = Optional.of(request.getThreshold()); } - if(request.hasIterations()) { + + if (request.hasIterations()) { iterations = Optional.of(request.getIterations()); } - if(request.hasMaxDatanodesRatioToInvolvePerIteration()) { - maxDatanodesRatioToInvolvePerIteration = - Optional.of(request.getMaxDatanodesRatioToInvolvePerIteration()); + + if (request.hasMaxDatanodesPercentageToInvolvePerIteration()) { + maxDatanodesPercentageToInvolvePerIteration = + Optional.of(request.getMaxDatanodesPercentageToInvolvePerIteration()); + } else if (request.hasMaxDatanodesRatioToInvolvePerIteration()) { + maxDatanodesPercentageToInvolvePerIteration = + Optional.of( + (int) (request.getMaxDatanodesRatioToInvolvePerIteration() * + 100)); } + if(request.hasMaxSizeToMovePerIterationInGB()) { maxSizeToMovePerIterationInGB = Optional.of(request.getMaxSizeToMovePerIterationInGB()); @@ -739,11 +748,9 @@ public StartContainerBalancerResponseProto startContainerBalancer( Optional.of(request.getMaxSizeLeavingSourceInGB()); } - - return StartContainerBalancerResponseProto.newBuilder(). setStart(impl.startContainerBalancer(threshold, - iterations, maxDatanodesRatioToInvolvePerIteration, + iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB)).build(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 22bc4cfe94ac..e5d79fd98a28 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -745,13 +745,13 @@ public StatusAndMessages queryUpgradeFinalizationProgress( @Override public boolean startContainerBalancer( Optional threshold, Optional iterations, - Optional maxDatanodesRatioToInvolvePerIteration, + Optional maxDatanodesPercentageToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTarget, Optional maxSizeLeavingSource) throws IOException { getScm().checkAdminAccess(getRemoteUser()); ContainerBalancerConfiguration cbc = - new ContainerBalancerConfiguration(scm.getConfiguration()); + scm.getConfiguration().getObject(ContainerBalancerConfiguration.class); if (threshold.isPresent()) { double tsd = threshold.get(); Preconditions.checkState(tsd >= 0.0D && tsd < 1.0D, @@ -764,21 +764,21 @@ public boolean startContainerBalancer( "maxSizeToMovePerIterationInGB must be positive."); cbc.setMaxSizeToMovePerIteration(mstm * OzoneConsts.GB); } - if (maxDatanodesRatioToInvolvePerIteration.isPresent()) { - double mdti = maxDatanodesRatioToInvolvePerIteration.get(); - Preconditions.checkState(mdti >= 0.0, - "maxDatanodesRatioToInvolvePerIteration must be " + + if (maxDatanodesPercentageToInvolvePerIteration.isPresent()) { + int mdti = maxDatanodesPercentageToInvolvePerIteration.get(); + Preconditions.checkState(mdti >= 0, + "maxDatanodesPercentageToInvolvePerIteration must be " + "greater than equal to zero."); - Preconditions.checkState(mdti <= 1, - "maxDatanodesRatioToInvolvePerIteration must be " + - "lesser than or equal to one."); - cbc.setMaxDatanodesRatioToInvolvePerIteration(mdti); + Preconditions.checkState(mdti <= 100, + "maxDatanodesPercentageToInvolvePerIteration must be " + + "lesser than or equal to 100."); + cbc.setMaxDatanodesPercentageToInvolvePerIteration(mdti); } if (iterations.isPresent()) { int i = iterations.get(); Preconditions.checkState(i > 0 || i == -1, - "iterations must be positive or" + - " -1(infinitly run container balancer)."); + "number of iterations must be positive or" + + " -1 (for running container balancer infinitely)."); cbc.setIterations(i); } @@ -807,7 +807,7 @@ public boolean startContainerBalancer( AUDIT.logWriteFailure(buildAuditMessageForSuccess( SCMAction.START_CONTAINER_BALANCER, null)); } - return isStartedSuccessfully; + return isStartedSuccessfully; } @Override diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java index 4d74c9af79cb..51ff529c01e8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java @@ -89,6 +89,7 @@ public class TestContainerBalancer { private Map> datanodeToContainersMap = new HashMap<>(); private static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current(); + private static final double DELTA = 1e-15; @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -102,10 +103,11 @@ public void setup() throws SCMException, NodeNotFoundException { containerManager = Mockito.mock(ContainerManager.class); replicationManager = Mockito.mock(ReplicationManager.class); - balancerConfiguration = new ContainerBalancerConfiguration(conf); + balancerConfiguration = + conf.getObject(ContainerBalancerConfiguration.class); balancerConfiguration.setThreshold(0.1); balancerConfiguration.setIterations(1); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); conf.setFromObject(balancerConfiguration); @@ -228,7 +230,9 @@ public void unBalancedNodesListShouldBeEmptyWhenClusterIsBalanced() { */ @Test public void containerBalancerShouldObeyMaxDatanodesToInvolveLimit() { - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(0.3d); + int percent = 20; + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration( + percent); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); balancerConfiguration.setThreshold(0.01); balancerConfiguration.setIterations(1); @@ -243,7 +247,7 @@ public void containerBalancerShouldObeyMaxDatanodesToInvolveLimit() { Assert.assertFalse( containerBalancer.getCountDatanodesInvolvedPerIteration() > - (int) (0.3 * numberOfNodes)); + (percent * numberOfNodes / 100)); containerBalancer.stop(); } @@ -317,7 +321,7 @@ public void containerBalancerShouldObeyMaxSizeToMoveLimit() { public void targetDatanodeShouldNotAlreadyContainSelectedContainer() { balancerConfiguration.setThreshold(0.1); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); containerBalancer.start(balancerConfiguration); // waiting for balance completed. @@ -344,7 +348,7 @@ public void targetDatanodeShouldNotAlreadyContainSelectedContainer() { public void containerMoveSelectionShouldFollowPlacementPolicy() { balancerConfiguration.setThreshold(0.1); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); containerBalancer.start(balancerConfiguration); // waiting for balance completed. @@ -385,7 +389,7 @@ public void containerMoveSelectionShouldFollowPlacementPolicy() { public void targetDatanodeShouldBeInServiceHealthy() throws NodeNotFoundException { balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); containerBalancer.start(balancerConfiguration); @@ -412,7 +416,7 @@ public void targetDatanodeShouldBeInServiceHealthy() @Test public void selectedContainerShouldNotAlreadyHaveBeenSelected() { balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); @@ -438,7 +442,7 @@ public void selectedContainerShouldNotAlreadyHaveBeenSelected() { @Test public void balancerShouldNotSelectConfiguredExcludeContainers() { balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); balancerConfiguration.setExcludeContainers("1, 4, 5"); @@ -465,7 +469,7 @@ public void balancerShouldNotSelectConfiguredExcludeContainers() { @Test public void balancerShouldObeyMaxSizeEnteringTargetLimit() { balancerConfiguration.setThreshold(0.1); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); // no containers should be selected when the limit is zero @@ -486,8 +490,10 @@ public void balancerShouldObeyMaxSizeEnteringTargetLimit() { Assert.assertTrue(containerBalancer.getSourceToTargetMap().isEmpty()); // some containers should be selected when using default values - containerBalancer.start( - new ContainerBalancerConfiguration(new OzoneConfiguration())); + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ContainerBalancerConfiguration cbc = ozoneConfiguration. + getObject(ContainerBalancerConfiguration.class); + containerBalancer.start(cbc); // waiting for balance completed. // TODO: this is a temporary implementation for now @@ -508,7 +514,7 @@ public void testMetrics() { balancerConfiguration.setIterations(1); balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); containerBalancer.start(balancerConfiguration); @@ -543,7 +549,7 @@ public void balancerShouldFollowExcludeAndIncludeDatanodesConfigurations() { balancerConfiguration.setIterations(1); balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); - balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0); + balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); // only these nodes should be included // the ones also specified in excludeNodes should be excluded @@ -589,6 +595,23 @@ public void balancerShouldFollowExcludeAndIncludeDatanodesConfigurations() { } } + @Test + public void testContainerBalancerConfiguration() { + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set("ozone.scm.container.size", "5GB"); + ozoneConfiguration.setDouble( + "hdds.container.balancer.utilization.threshold", 0.01); + + ContainerBalancerConfiguration cbConf = + ozoneConfiguration.getObject(ContainerBalancerConfiguration.class); + Assert.assertEquals(cbConf.getThreshold(), 0.01d, DELTA); + + Assert.assertEquals(cbConf.getMaxSizeLeavingSource(), + 26 * 1024 * 1024 * 1024L); + + Assert.assertEquals(cbConf.getMoveTimeout().toMillis(), 30 * 60 * 1000); + } + /** * Determines unBalanced nodes, that is, over and under utilized nodes, * according to the generated utilization values for nodes and the threshold. diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java index 3bea380b1882..7f24d843b0f1 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java @@ -42,23 +42,22 @@ * ozone admin containerbalancer start * [ -t/--threshold {@literal }] * [ -i/--iterations {@literal }] - * [ -d/--maxDatanodesRatioToInvolvePerIteration - * {@literal }] + * [ -d/--maxDatanodesPercentageToInvolvePerIteration + * {@literal }] * [ -s/--maxSizeToMovePerIterationInGB * {@literal }] * Examples: * ozone admin containerbalancer start * start balancer with default values in the configuration - * ozone admin containerbalancer start -t 0.05 + * ozone admin containerbalancer start -t 5 * start balancer with a threshold of 5% * ozone admin containerbalancer start -i 20 * start balancer with maximum 20 consecutive iterations - * ozone admin containerbalancer start -i 0 + * ozone admin containerbalancer start -i -1 * run balancer infinitely with default values in the configuration - * ozone admin containerbalancer start -d 0.4 - * start balancer with the ratio of maximum datanodes to involve in - * balancing in one iteration to the total number of healthy datanodes as - * 0.4 + * ozone admin containerbalancer start -d 40 + * start balancer with maximum 40% of healthy, in-service datanodes + * involved in balancing * ozone admin containerbalancer start -s 10 * start balancer with maximum size of 10GB to move in one iteration * To stop: diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index a8820d5c7c6c..c550a5e0836f 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -44,11 +44,11 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { " balancer will run for") private Optional iterations; - @Option(names = {"-d", "--maxDatanodesRatioToInvolvePerIteration"}, - description = "The ratio of maximum number of datanodes that should be " + - "involved in balancing in one iteration to the total number of " + - "healthy, in service nodes known to container balancer.") - private Optional maxDatanodesRatioToInvolvePerIteration; + @Option(names = {"-d", "--maxDatanodesPercentageToInvolvePerIteration"}, + description = "The max percentage of healthy, in service datanodes " + + "that can be involved in balancing in one iteration.") + private Optional maxDatanodesPercentageToInvolvePerIteration; + @Option(names = {"-s", "--maxSizeToMovePerIterationInGB"}, description = "Maximum size to move per iteration of balancing in GB, " + @@ -68,8 +68,9 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { @Override public void execute(ScmClient scmClient) throws IOException { boolean result = scmClient.startContainerBalancer(threshold, iterations, - maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, - maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); + maxDatanodesPercentageToInvolvePerIteration, + maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, + maxSizeLeavingSourceInGB); if (result) { System.out.println("Starting ContainerBalancer Successfully."); return; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 182c77e39ab0..123a74d70920 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -553,13 +553,13 @@ public boolean getReplicationManagerStatus() throws IOException { @Override public boolean startContainerBalancer( Optional threshold, Optional iterations, - Optional maxDatanodesRatioToInvolvePerIteration, + Optional maxDatanodesPercentageToInvolvePerIteration, Optional maxSizeToMovePerIterationInGB, Optional maxSizeEnteringTargetInGB, Optional maxSizeLeavingSourceInGB) throws IOException { return storageContainerLocationClient.startContainerBalancer(threshold, - iterations, maxDatanodesRatioToInvolvePerIteration, + iterations, maxDatanodesPercentageToInvolvePerIteration, maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java index 9a84453cd0b6..2b0dfc2033ac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java @@ -82,14 +82,16 @@ public void testContainerBalancerCLIOperations() throws Exception { assertFalse(running); Optional threshold = Optional.of(0.1); Optional iterations = Optional.of(10000); - Optional maxDatanodesRatioToInvolvePerIteration = Optional.of(1d); + Optional maxDatanodesPercentageToInvolvePerIteration = + Optional.of(100); Optional maxSizeToMovePerIterationInGB = Optional.of(1L); Optional maxSizeEnteringTargetInGB = Optional.of(1L); Optional maxSizeLeavingSourceInGB = Optional.of(1L); containerBalancerClient.startContainerBalancer(threshold, iterations, - maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, - maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); + maxDatanodesPercentageToInvolvePerIteration, + maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, + maxSizeLeavingSourceInGB); running = containerBalancerClient.getContainerBalancerStatus(); assertTrue(running); @@ -105,8 +107,9 @@ public void testContainerBalancerCLIOperations() throws Exception { // test normally start , and stop it before balance is completed containerBalancerClient.startContainerBalancer(threshold, iterations, - maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB, - maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); + maxDatanodesPercentageToInvolvePerIteration, + maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, + maxSizeLeavingSourceInGB); running = containerBalancerClient.getContainerBalancerStatus(); assertTrue(running); From 608b5e5bce08070bd1969900da97520ebca08a08 Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Wed, 22 Dec 2021 23:25:51 +0530 Subject: [PATCH 3/6] represent threshold as a percentage instead a ratio --- .../container/balancer/ContainerBalancer.java | 2 +- .../ContainerBalancerConfiguration.java | 20 +++++---- .../scm/server/SCMClientProtocolServer.java | 4 +- .../balancer/TestContainerBalancer.java | 42 ++++++++++--------- .../cli/ContainerBalancerStartSubcommand.java | 34 ++++++++------- .../TestContainerBalancerSubCommand.java | 8 ++-- 6 files changed, 60 insertions(+), 50 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index d02b09429fbc..1b69098b3c2a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -245,7 +245,7 @@ private boolean initializeIteration() { } return false; } - this.threshold = config.getThreshold(); + this.threshold = config.getThresholdAsRatio(); this.maxDatanodesRatioToInvolvePerIteration = config.getMaxDatanodesRatioToInvolvePerIteration(); this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index 0645a2193c10..e211c7abb1a6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -43,13 +43,13 @@ public final class ContainerBalancerConfiguration { LoggerFactory.getLogger(ContainerBalancerConfiguration.class); @Config(key = "utilization.threshold", type = ConfigType.AUTO, defaultValue = - "0.1", tags = {ConfigTag.BALANCER}, - description = "Threshold is a fraction in the range of 0 to 1. A " + + "10", tags = {ConfigTag.BALANCER}, + description = "Threshold is a percentage in the range of 0 to 100. A " + "cluster is considered balanced if for each datanode, the " + "utilization of the datanode (used space to capacity ratio) differs" + " from the utilization of the cluster (used space to capacity ratio" + - " of the entire cluster) no more than the threshold value.") - private String threshold = "0.1"; + " of the entire cluster) no more than the threshold.") + private String threshold = "10"; @Config(key = "datanodes.involved.max.percentage.per.iteration", type = ConfigType.INT, defaultValue = "20", tags = {ConfigTag.BALANCER}, @@ -126,21 +126,25 @@ public final class ContainerBalancerConfiguration { /** * Gets the threshold value for Container Balancer. * - * @return a fraction in the range 0 to 1 + * @return percentage value in the range 0 to 100 */ public double getThreshold() { return Double.parseDouble(threshold); } + public double getThresholdAsRatio() { + return Double.parseDouble(threshold) / 100; + } + /** * Sets the threshold value for Container Balancer. * - * @param threshold a fraction in the range 0 to 1 + * @param threshold a percentage value in the range 0 to 100 */ public void setThreshold(double threshold) { - if (threshold < 0 || threshold > 1) { + if (threshold < 0d || threshold >= 100d) { throw new IllegalArgumentException( - "Threshold must be a fraction in the range 0 to 1."); + "Threshold must be a percentage(double) in the range 0 to 100."); } this.threshold = String.valueOf(threshold); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index e5d79fd98a28..ac2c4b0fe1fd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -754,8 +754,8 @@ public boolean startContainerBalancer( scm.getConfiguration().getObject(ContainerBalancerConfiguration.class); if (threshold.isPresent()) { double tsd = threshold.get(); - Preconditions.checkState(tsd >= 0.0D && tsd < 1.0D, - "threshold should to be specified in range [0.0, 1.0)."); + Preconditions.checkState(tsd >= 0.0D && tsd < 100.0D, + "threshold should to be specified in range [0.0, 1.000)."); cbc.setThreshold(tsd); } if (maxSizeToMovePerIterationInGB.isPresent()) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java index 4c984ca2fdb0..501ca1914d96 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java @@ -106,7 +106,7 @@ public void setup() throws SCMException, NodeNotFoundException { balancerConfiguration = conf.getObject(ContainerBalancerConfiguration.class); - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setIterations(1); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); @@ -177,7 +177,7 @@ public void testCalculationOfUtilization() { // check for random threshold values for (int i = 0; i < 50; i++) { - double randomThreshold = RANDOM.nextDouble(); + double randomThreshold = RANDOM.nextDouble() * 100; balancerConfiguration.setThreshold(randomThreshold); containerBalancer.start(balancerConfiguration); @@ -212,7 +212,7 @@ public void testCalculationOfUtilization() { */ @Test public void unBalancedNodesListShouldBeEmptyWhenClusterIsBalanced() { - balancerConfiguration.setThreshold(0.99); + balancerConfiguration.setThreshold(99.99); containerBalancer.start(balancerConfiguration); // waiting for balance completed. @@ -236,7 +236,7 @@ public void containerBalancerShouldObeyMaxDatanodesToInvolveLimit() { balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration( percent); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); - balancerConfiguration.setThreshold(0.01); + balancerConfiguration.setThreshold(1); balancerConfiguration.setIterations(1); containerBalancer.start(balancerConfiguration); @@ -259,7 +259,7 @@ public void containerBalancerShouldSelectOnlyClosedContainers() { for (ContainerInfo containerInfo : cidToInfoMap.values()) { containerInfo.setState(HddsProtos.LifeCycleState.OPEN); } - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); containerBalancer.start(balancerConfiguration); // waiting for balance completed. @@ -301,7 +301,7 @@ public void containerBalancerShouldSelectOnlyClosedContainers() { @Test public void containerBalancerShouldObeyMaxSizeToMoveLimit() { - balancerConfiguration.setThreshold(0.01); + balancerConfiguration.setThreshold(1); balancerConfiguration.setMaxSizeToMovePerIteration(10 * OzoneConsts.GB); balancerConfiguration.setIterations(1); containerBalancer.start(balancerConfiguration); @@ -321,7 +321,7 @@ public void containerBalancerShouldObeyMaxSizeToMoveLimit() { @Test public void targetDatanodeShouldNotAlreadyContainSelectedContainer() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); containerBalancer.start(balancerConfiguration); @@ -348,7 +348,7 @@ public void targetDatanodeShouldNotAlreadyContainSelectedContainer() { @Test public void containerMoveSelectionShouldFollowPlacementPolicy() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); containerBalancer.start(balancerConfiguration); @@ -390,7 +390,7 @@ public void containerMoveSelectionShouldFollowPlacementPolicy() { @Test public void targetDatanodeShouldBeInServiceHealthy() throws NodeNotFoundException { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); @@ -417,7 +417,7 @@ public void targetDatanodeShouldBeInServiceHealthy() @Test public void selectedContainerShouldNotAlreadyHaveBeenSelected() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); @@ -443,7 +443,7 @@ public void selectedContainerShouldNotAlreadyHaveBeenSelected() { @Test public void balancerShouldNotSelectConfiguredExcludeContainers() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB); @@ -470,7 +470,7 @@ public void balancerShouldNotSelectConfiguredExcludeContainers() { @Test public void balancerShouldObeyMaxSizeEnteringTargetLimit() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setMaxDatanodesPercentageToInvolvePerIteration(100); balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB); @@ -512,7 +512,7 @@ public void balancerShouldObeyMaxSizeEnteringTargetLimit() { @Test public void testMetrics() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setIterations(1); balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); @@ -547,7 +547,7 @@ public void testMetrics() { */ @Test public void balancerShouldFollowExcludeAndIncludeDatanodesConfigurations() { - balancerConfiguration.setThreshold(0.1); + balancerConfiguration.setThreshold(10); balancerConfiguration.setIterations(1); balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB); balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB); @@ -602,28 +602,30 @@ public void testContainerBalancerConfiguration() { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set("ozone.scm.container.size", "5GB"); ozoneConfiguration.setDouble( - "hdds.container.balancer.utilization.threshold", 0.01); + "hdds.container.balancer.utilization.threshold", 1); ContainerBalancerConfiguration cbConf = ozoneConfiguration.getObject(ContainerBalancerConfiguration.class); - Assert.assertEquals(cbConf.getThreshold(), 0.01d, DELTA); + Assert.assertEquals(1, cbConf.getThreshold(), DELTA); - Assert.assertEquals(cbConf.getMaxSizeLeavingSource(), - 26 * 1024 * 1024 * 1024L); + Assert.assertEquals(26 * 1024 * 1024 * 1024L, + cbConf.getMaxSizeLeavingSource()); - Assert.assertEquals(cbConf.getMoveTimeout().toMillis(), 30 * 60 * 1000); + Assert.assertEquals(30 * 60 * 1000, + cbConf.getMoveTimeout().toMillis()); } /** * Determines unBalanced nodes, that is, over and under utilized nodes, * according to the generated utilization values for nodes and the threshold. * - * @param threshold A fraction from range 0 to 1. + * @param threshold A percentage in the range 0 to 100 * @return List of DatanodeUsageInfo containing the expected(correct) * unBalanced nodes. */ private List determineExpectedUnBalancedNodes( double threshold) { + threshold /= 100; double lowerLimit = averageUtilization - threshold; double upperLimit = averageUtilization + threshold; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index c550a5e0836f..73d8511bb0ca 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -36,33 +36,37 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { @Option(names = {"-t", "--threshold"}, - description = "Threshold target whether the cluster is balanced") + description = "Percentage deviation from average utilization of " + + "the cluster after which a datanode will be rebalanced (for " + + "example, '10' for 10%).") private Optional threshold; @Option(names = {"-i", "--iterations"}, description = "Maximum consecutive iterations that" + - " balancer will run for") + " balancer will run for.") private Optional iterations; @Option(names = {"-d", "--maxDatanodesPercentageToInvolvePerIteration"}, - description = "The max percentage of healthy, in service datanodes " + - "that can be involved in balancing in one iteration.") + description = "Max percentage of healthy, in service datanodes " + + "that can be involved in balancing in one iteration (for example, " + + "'20' for 20%).") private Optional maxDatanodesPercentageToInvolvePerIteration; - @Option(names = {"-s", "--maxSizeToMovePerIterationInGB"}, - description = "Maximum size to move per iteration of balancing in GB, " + - "for 10GB it should be set as 10") + description = "Maximum size that can be moved per iteration of " + + "balancing (for example, '500' for 500GB).") private Optional maxSizeToMovePerIterationInGB; - @Option(names = {"-e", "--maxSizeEnteringTarget"}, - description = "the maximum size that can enter a target datanode while " + - "balancing in GB. This is the sum of data from multiple sources.") + @Option(names = {"-e", "--maxSizeEnteringTargetInGB"}, + description = "Maximum size that can enter a target datanode while " + + "balancing. This is the sum of data from multiple sources (for " + + "example, '26' for 26GB).") private Optional maxSizeEnteringTargetInGB; - @Option(names = {"-l", "--maxSizeLeavingSource"}, - description = "maximum size that can leave a source datanode while " + - "balancing in GB, it is the sum of data moving to multiple targets.") + @Option(names = {"-l", "--maxSizeLeavingSourceInGB"}, + description = "Maximum size that can leave a source datanode while " + + "balancing. This is the sum of data moving to multiple targets " + + "(for example, '26' for 26GB).") private Optional maxSizeLeavingSourceInGB; @Override @@ -72,10 +76,10 @@ public void execute(ScmClient scmClient) throws IOException { maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB); if (result) { - System.out.println("Starting ContainerBalancer Successfully."); + System.out.println("Container Balancer started successfully."); return; } - System.out.println("ContainerBalancer is already running, " + + System.out.println("Container Balancer is already running. " + "Please stop it first."); } } \ No newline at end of file diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index 8ed6acd7b2bf..a7ef77996a47 100644 --- a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java @@ -119,8 +119,8 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsNotRunning() .thenAnswer(invocation -> true); startCmd.execute(scmClient); - Pattern p = Pattern.compile("^Starting\\sContainerBalancer" + - "\\sSuccessfully."); + Pattern p = Pattern.compile("^Container\\sBalancer\\sstarted" + + "\\ssuccessfully."); Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING)); assertTrue(m.find()); } @@ -134,8 +134,8 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsRunning() .thenAnswer(invocation -> false); startCmd.execute(scmClient); - Pattern p = Pattern.compile("^ContainerBalancer\\sis\\salready\\srunning," + - "\\sPlease\\sstop\\sit\\sfirst."); + Pattern p = Pattern.compile("^Container\\sBalancer\\sis\\salready" + + "\\srunning.\\sPlease\\sstop\\sit\\sfirst."); Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING)); assertTrue(m.find()); } From 63117c53c89536989af80217fe6c964c1e245fbf Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Thu, 23 Dec 2021 11:33:12 +0530 Subject: [PATCH 4/6] get ContainerBalancerConfiguration from OzoneConfiguration in the ContainerBalancer constructor. This change is required for ContainerBalancer#toString() to be able to call ContainerBalancerConfiguration#toString(). --- .../hadoop/hdds/scm/container/balancer/ContainerBalancer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 1b69098b3c2a..24c79389bf30 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -125,6 +125,7 @@ public ContainerBalancer( this.containerManager = containerManager; this.replicationManager = replicationManager; this.ozoneConfiguration = ozoneConfiguration; + config = ozoneConfiguration.getObject(ContainerBalancerConfiguration.class); this.metrics = ContainerBalancerMetrics.create(); this.scmContext = scmContext; From 2a46d525bdec305f750561d35549d88cf551bb6e Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Fri, 24 Dec 2021 11:04:43 +0530 Subject: [PATCH 5/6] minor fixes --- .../hadoop/hdds/scm/container/balancer/ContainerBalancer.java | 2 +- .../scm/container/balancer/ContainerBalancerConfiguration.java | 2 +- .../apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java index 24c79389bf30..386fb761438c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java @@ -866,7 +866,7 @@ int getCountDatanodesInvolvedPerIteration() { return countDatanodesInvolvedPerIteration; } - long getSizeMovedPerIteration() { + public long getSizeMovedPerIteration() { return sizeMovedPerIteration; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index e211c7abb1a6..858234785e69 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -92,7 +92,7 @@ public final class ContainerBalancerConfiguration { @Config(key = "move.timeout", type = ConfigType.TIME, defaultValue = "30m", tags = {ConfigTag.BALANCER}, description = - "The amount of time in minutes to allow a single container to move " + + "The amount of time to allow a single container to move " + "from source to target.") private long moveTimeout = Duration.ofMinutes(30).toMillis(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index ac2c4b0fe1fd..a1dbfa141aba 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -755,7 +755,7 @@ public boolean startContainerBalancer( if (threshold.isPresent()) { double tsd = threshold.get(); Preconditions.checkState(tsd >= 0.0D && tsd < 100.0D, - "threshold should to be specified in range [0.0, 1.000)."); + "threshold should to be specified in range [0.0, 100.0)."); cbc.setThreshold(tsd); } if (maxSizeToMovePerIterationInGB.isPresent()) { From a692564463f10d72bba32ccee8e1ae214dfb5f2d Mon Sep 17 00:00:00 2001 From: Siddhant Sangwan Date: Mon, 10 Jan 2022 19:02:34 +0530 Subject: [PATCH 6/6] handle idleIterations and correct threshold range --- ...torageContainerLocationProtocolClientSideTranslatorPB.java | 4 ++-- ...torageContainerLocationProtocolServerSideTranslatorPB.java | 2 ++ .../hadoop/hdds/scm/server/SCMClientProtocolServer.java | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index e8b8c246c668..b7458222c819 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -751,8 +751,8 @@ public boolean startContainerBalancer( //make balancer configuration optional if (threshold.isPresent()) { double tsd = threshold.get(); - Preconditions.checkState(tsd >= 0.0D && tsd < 1.0D, - "threshold should to be specified in range [0.0, 1.0)."); + Preconditions.checkState(tsd >= 0.0D && tsd < 100D, + "threshold should be specified in range [0.0, 100.0)."); builder.setThreshold(tsd); } if (maxSizeToMovePerIterationInGB.isPresent()) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 2eba37f00a41..b839e991da16 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -731,6 +731,8 @@ public StartContainerBalancerResponseProto startContainerBalancer( if (request.hasIterations()) { iterations = Optional.of(request.getIterations()); + } else if (request.hasIdleiterations()) { + iterations = Optional.of(request.getIdleiterations()); } if (request.hasMaxDatanodesPercentageToInvolvePerIteration()) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index c5ff3e3e94dd..fa1d316d99db 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -755,7 +755,7 @@ public boolean startContainerBalancer( if (threshold.isPresent()) { double tsd = threshold.get(); Preconditions.checkState(tsd >= 0.0D && tsd < 100.0D, - "threshold should to be specified in range [0.0, 100.0)."); + "threshold should be specified in range [0.0, 100.0)."); cbc.setThreshold(tsd); } if (maxSizeToMovePerIterationInGB.isPresent()) {