HDDS-5927. Improve defaults in ContainerBalancerConfiguration

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java

@@ -312,7 +312,7 @@ Map<String, Pair<Boolean, String>> getSafeModeRuleStatuses()
    * Start ContainerBalancer.
    */
   boolean startContainerBalancer(Optional<Double> threshold,
-      Optional<Integer> idleiterations,
+      Optional<Integer> iterations,
       Optional<Double> maxDatanodesRatioToInvolvePerIteration,
       Optional<Long> maxSizeToMovePerIterationInGB,
       Optional<Long> maxSizeEnteringTargetInGB,

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java

@@ -312,7 +312,7 @@ Map<String, Pair<Boolean, String>> getSafeModeRuleStatuses()
    * Start ContainerBalancer.
    */
   boolean startContainerBalancer(Optional<Double> threshold,
-      Optional<Integer> idleiterations,
+      Optional<Integer> iterations,
       Optional<Double> maxDatanodesRatioToInvolvePerIteration,
       Optional<Long> maxSizeToMovePerIterationInGB,
       Optional<Long> maxSizeEnteringTargetInGB,

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -737,7 +737,7 @@ public boolean getReplicationManagerStatus() throws IOException {
 
   @Override
   public boolean startContainerBalancer(
-      Optional<Double> threshold, Optional<Integer> idleiterations,
+      Optional<Double> threshold, Optional<Integer> iterations,
       Optional<Double> maxDatanodesRatioToInvolvePerIteration,
       Optional<Long> maxSizeToMovePerIterationInGB,
       Optional<Long> maxSizeEnteringTargetInGB,
@@ -769,12 +769,12 @@ public boolean startContainerBalancer(
               "lesser than equal to one.");
       builder.setMaxDatanodesRatioToInvolvePerIteration(mdti);
     }
-    if (idleiterations.isPresent()) {
-      int idi = idleiterations.get();
-      Preconditions.checkState(idi > 0 || idi == -1,
-          "idleiterations must be positive or" +
+    if (iterations.isPresent()) {
+      int i = iterations.get();
+      Preconditions.checkState(i > 0 || i == -1,
+          "iterations must be positive or" +
               " -1(infinitly run container balancer).");
-      builder.setIdleiterations(idi);
+      builder.setIterations(i);
     }
 
     if (maxSizeEnteringTargetInGB.isPresent()) {

hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto

@@ -479,7 +479,7 @@ message GetContainerTokenResponseProto {
 message StartContainerBalancerRequestProto {
   optional string traceID = 1;
   optional double threshold = 2;
-  optional int32 idleiterations = 3;
+  optional int32 iterations = 3;
   optional double maxDatanodesRatioToInvolvePerIteration = 4;
   optional int64 maxSizeToMovePerIterationInGB = 5;
   optional int64 maxSizeEnteringTargetInGB = 6;

hadoop-hdds/interface-admin/src/main/resources/proto.lock

@@ -1334,7 +1334,7 @@
               },
               {
                 "id": 3,
-                "name": "idleiterations",
+                "name": "iterations",
                 "type": "int32"
               },
               {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java

@@ -72,7 +72,7 @@ public class ContainerBalancer {
   private long maxSizeToMovePerIteration;
   private int countDatanodesInvolvedPerIteration;
   private long sizeMovedPerIteration;
-  private int idleIteration;
+  private int iterations;
   private List<DatanodeUsageInfo> unBalancedNodes;
   private List<DatanodeUsageInfo> overUtilizedNodes;
   private List<DatanodeUsageInfo> underUtilizedNodes;
@@ -170,16 +170,16 @@ public boolean start(ContainerBalancerConfiguration balancerConfiguration) {
    * Balances the cluster.
    */
   private void balance() {
-    this.idleIteration = config.getIdleIteration();
-    if(this.idleIteration == -1) {
+    this.iterations = config.getIterations();
+    if(this.iterations == -1) {
       //run balancer infinitely
-      this.idleIteration = Integer.MAX_VALUE;
+      this.iterations = Integer.MAX_VALUE;
     }
     this.threshold = config.getThreshold();
     this.maxDatanodesRatioToInvolvePerIteration =
         config.getMaxDatanodesRatioToInvolvePerIteration();
     this.maxSizeToMovePerIteration = config.getMaxSizeToMovePerIteration();
-    for (int i = 0; i < idleIteration && balancerRunning; i++) {
+    for (int i = 0; i < iterations && balancerRunning; i++) {
       // stop balancing if iteration is not initialized
       if (!initializeIteration()) {
         stop();
@@ -202,7 +202,7 @@ private void balance() {
 
       // wait for configured time before starting next iteration, unless
       // this was the final iteration
-      if (i != idleIteration - 1) {
+      if (i != iterations - 1) {
         synchronized (this) {
           try {
             wait(config.getBalancingInterval().toMillis());

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java

@@ -76,22 +76,23 @@ public final class ContainerBalancerConfiguration {
       defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " +
       "maximum size that can enter a target datanode in each " +
       "iteration while balancing. This is the sum of data from multiple " +
-      "sources. The default value is greater than the configured" +
-      " (or default) ozone.scm.container.size by 1GB.")
+      "sources. By default, five times the configured (or default) ozone.scm" +
+      ".container.size is allowed.")
   private long maxSizeEnteringTarget;
 
   @Config(key = "size.leaving.source.max", type = ConfigType.SIZE,
       defaultValue = "", tags = {ConfigTag.BALANCER}, description = "The " +
       "maximum size that can leave a source datanode in each " +
       "iteration while balancing. This is the sum of data moving to multiple " +
-      "targets. The default value is greater than the configured" +
-      " (or default) ozone.scm.container.size by 1GB.")
+      "targets. By default, five times the configured (or default) ozone.scm" +
+      ".container.size is allowed.")
   private long maxSizeLeavingSource;
 
-  @Config(key = "idle.iterations", type = ConfigType.INT,
+  @Config(key = "iterations", type = ConfigType.INT,
       defaultValue = "10", tags = {ConfigTag.BALANCER},
-      description = "The idle iteration count of Container Balancer.")
-  private int idleIterations = 10;
+      description = "The number of iterations that Container Balancer will " +
+          "run for.")
+  private int iterations = 10;
 
   @Config(key = "exclude.containers", type = ConfigType.STRING, defaultValue =
       "", tags = {ConfigTag.BALANCER}, description = "List of container IDs " +
@@ -136,14 +137,13 @@ public ContainerBalancerConfiguration(OzoneConfiguration config) {
         "OzoneConfiguration should not be null.");
     this.ozoneConfiguration = config;
 
-    // maxSizeEnteringTarget and maxSizeLeavingSource should by default be
-    // greater than container size
-    long size = (long) ozoneConfiguration.getStorageSize(
+    // maxSizeEnteringTarget and maxSizeLeavingSource should by default allow
+    // five containers
+    long size = 5 * (long) ozoneConfiguration.getStorageSize(
         ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.GB) +
-        OzoneConsts.GB;
-    maxSizeEnteringTarget = size;
-    maxSizeLeavingSource = size;
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+    maxSizeEnteringTarget = size + OzoneConsts.GB;
+    maxSizeLeavingSource = size + OzoneConsts.GB;
 
     // balancing interval should be greater than DUFactory refresh period
     duConf = ozoneConfiguration.getObject(DUFactory.Conf.class);
@@ -174,26 +174,27 @@ public void setThreshold(double threshold) {
   }
 
   /**
-   * Gets the idle iteration value for Container Balancer.
+   * Gets the iteration count for Container Balancer. A value of -1 means
+   * infinite number of iterations.
    *
-   * @return a idle iteration count larger than 0
+   * @return a value greater than 0, or -1
    */
-  public int getIdleIteration() {
-    return idleIterations;
+  public int getIterations() {
+    return iterations;
   }
 
   /**
-   * Sets the idle iteration value for Container Balancer.
+   * Sets the number of iterations for Container Balancer.
    *
-   * @param count a idle iteration count larger than 0
+   * @param count a value greater than 0, or -1
    */
-  public void setIdleIteration(int count) {
+  public void setIterations(int count) {
     if (count < -1 || 0 == count) {
       throw new IllegalArgumentException(
-          "Idle iteration count must be larger than 0 or " +
-              "-1(for infinitely running).");
+          "Iteration count must be greater than 0, or " +
+              "-1(for running balancer infinitely).");
     }
-    this.idleIterations = count;
+    this.iterations = count;
   }
 
   /**

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java

@@ -710,7 +710,7 @@ public StartContainerBalancerResponseProto startContainerBalancer(
       StartContainerBalancerRequestProto request)
       throws IOException {
     Optional<Double> threshold = Optional.empty();
-    Optional<Integer> idleiterations = Optional.empty();
+    Optional<Integer> iterations = Optional.empty();
     Optional<Double> maxDatanodesRatioToInvolvePerIteration = Optional.empty();
     Optional<Long> maxSizeToMovePerIterationInGB = Optional.empty();
     Optional<Long> maxSizeEnteringTargetInGB = Optional.empty();
@@ -719,8 +719,8 @@ public StartContainerBalancerResponseProto startContainerBalancer(
     if(request.hasThreshold()) {
       threshold = Optional.of(request.getThreshold());
     }
-    if(request.hasIdleiterations()) {
-      idleiterations = Optional.of(request.getIdleiterations());
+    if(request.hasIterations()) {
+      iterations = Optional.of(request.getIterations());
     }
     if(request.hasMaxDatanodesRatioToInvolvePerIteration()) {
       maxDatanodesRatioToInvolvePerIteration =
@@ -743,7 +743,7 @@ public StartContainerBalancerResponseProto startContainerBalancer(
 
     return StartContainerBalancerResponseProto.newBuilder().
         setStart(impl.startContainerBalancer(threshold,
-            idleiterations, maxDatanodesRatioToInvolvePerIteration,
+            iterations, maxDatanodesRatioToInvolvePerIteration,
             maxSizeToMovePerIterationInGB,
             maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB)).build();
   }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -744,7 +744,7 @@ public StatusAndMessages queryUpgradeFinalizationProgress(
 
   @Override
   public boolean startContainerBalancer(
-      Optional<Double> threshold, Optional<Integer> idleiterations,
+      Optional<Double> threshold, Optional<Integer> iterations,
       Optional<Double> maxDatanodesRatioToInvolvePerIteration,
       Optional<Long> maxSizeToMovePerIterationInGB,
       Optional<Long> maxSizeEnteringTarget,
@@ -774,12 +774,12 @@ public boolean startContainerBalancer(
               "lesser than or equal to one.");
       cbc.setMaxDatanodesRatioToInvolvePerIteration(mdti);
     }
-    if (idleiterations.isPresent()) {
-      int idi = idleiterations.get();
-      Preconditions.checkState(idi > 0 || idi == -1,
-          "idleiterations must be positive or" +
+    if (iterations.isPresent()) {
+      int i = iterations.get();
+      Preconditions.checkState(i > 0 || i == -1,
+          "iterations must be positive or" +
               " -1(infinitly run container balancer).");
-      cbc.setIdleIteration(idi);
+      cbc.setIterations(i);
     }
 
     if (maxSizeEnteringTarget.isPresent()) {

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancer.java

@@ -104,7 +104,7 @@ public void setup() throws SCMException, NodeNotFoundException {
 
     balancerConfiguration = new ContainerBalancerConfiguration(conf);
     balancerConfiguration.setThreshold(0.1);
-    balancerConfiguration.setIdleIteration(1);
+    balancerConfiguration.setIterations(1);
     balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0d);
     balancerConfiguration.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
     balancerConfiguration.setMaxSizeEnteringTarget(5 * OzoneConsts.GB);
@@ -231,7 +231,7 @@ public void containerBalancerShouldObeyMaxDatanodesToInvolveLimit() {
     balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(0.3d);
     balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
     balancerConfiguration.setThreshold(0.01);
-    balancerConfiguration.setIdleIteration(1);
+    balancerConfiguration.setIterations(1);
     containerBalancer.start(balancerConfiguration);
 
     // waiting for balance completed.
@@ -297,7 +297,7 @@ public void containerBalancerShouldSelectOnlyClosedContainers() {
   public void containerBalancerShouldObeyMaxSizeToMoveLimit() {
     balancerConfiguration.setThreshold(0.01);
     balancerConfiguration.setMaxSizeToMovePerIteration(10 * OzoneConsts.GB);
-    balancerConfiguration.setIdleIteration(1);
+    balancerConfiguration.setIterations(1);
     containerBalancer.start(balancerConfiguration);
 
     // waiting for balance completed.
@@ -505,7 +505,7 @@ public void balancerShouldObeyMaxSizeEnteringTargetLimit() {
   @Test
   public void testMetrics() {
     balancerConfiguration.setThreshold(0.1);
-    balancerConfiguration.setIdleIteration(1);
+    balancerConfiguration.setIterations(1);
     balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB);
     balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
     balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0);
@@ -540,7 +540,7 @@ public void testMetrics() {
   @Test
   public void balancerShouldFollowExcludeAndIncludeDatanodesConfigurations() {
     balancerConfiguration.setThreshold(0.1);
-    balancerConfiguration.setIdleIteration(1);
+    balancerConfiguration.setIterations(1);
     balancerConfiguration.setMaxSizeEnteringTarget(10 * OzoneConsts.GB);
     balancerConfiguration.setMaxSizeToMovePerIteration(100 * OzoneConsts.GB);
     balancerConfiguration.setMaxDatanodesRatioToInvolvePerIteration(1.0);

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java

@@ -41,7 +41,7 @@
  * To start:
  *      ozone admin containerbalancer start
  *      [ -t/--threshold {@literal <threshold>}]
- *      [ -i/--idleiterations {@literal <idleiterations>}]
+ *      [ -i/--iterations {@literal <iterations>}]
  *      [ -d/--maxDatanodesRatioToInvolvePerIteration
  *      {@literal <maxDatanodesRatioToInvolvePerIteration>}]
  *      [ -s/--maxSizeToMovePerIterationInGB
@@ -52,7 +52,7 @@
  *      ozone admin containerbalancer start -t 0.05
  *        start balancer with a threshold of 5%
  *      ozone admin containerbalancer start -i 20
- *        start balancer with maximum 20 consecutive idle iterations
+ *        start balancer with maximum 20 consecutive iterations
  *      ozone admin containerbalancer start -i 0
  *        run balancer infinitely with default values in the configuration
  *      ozone admin containerbalancer start -d 0.4

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java

@@ -39,9 +39,10 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand {
       description = "Threshold target whether the cluster is balanced")
   private Optional<Double> threshold;
 
-  @Option(names = {"-i", "--idleiterations"},
-      description = "Maximum consecutive idle iterations")
-  private Optional<Integer> idleiterations;
+  @Option(names = {"-i", "--iterations"},
+      description = "Maximum consecutive iterations that" +
+          " balancer will run for")
+  private Optional<Integer> iterations;
 
   @Option(names = {"-d", "--maxDatanodesRatioToInvolvePerIteration"},
       description = "The ratio of maximum number of datanodes that should be " +
@@ -66,7 +67,7 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand {
 
   @Override
   public void execute(ScmClient scmClient) throws IOException {
-    boolean result = scmClient.startContainerBalancer(threshold, idleiterations,
+    boolean result = scmClient.startContainerBalancer(threshold, iterations,
         maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB,
         maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB);
     if (result) {

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java

@@ -552,14 +552,14 @@ public boolean getReplicationManagerStatus() throws IOException {
 
   @Override
   public boolean startContainerBalancer(
-      Optional<Double> threshold, Optional<Integer> idleiterations,
+      Optional<Double> threshold, Optional<Integer> iterations,
       Optional<Double> maxDatanodesRatioToInvolvePerIteration,
       Optional<Long> maxSizeToMovePerIterationInGB,
       Optional<Long> maxSizeEnteringTargetInGB,
       Optional<Long> maxSizeLeavingSourceInGB)
       throws IOException {
     return storageContainerLocationClient.startContainerBalancer(threshold,
-        idleiterations, maxDatanodesRatioToInvolvePerIteration,
+        iterations, maxDatanodesRatioToInvolvePerIteration,
         maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB,
         maxSizeLeavingSourceInGB);
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerBalancerOperations.java

@@ -81,13 +81,13 @@ public void testContainerBalancerCLIOperations() throws Exception {
     boolean running = containerBalancerClient.getContainerBalancerStatus();
     assertFalse(running);
     Optional<Double> threshold = Optional.of(0.1);
-    Optional<Integer> idleiterations = Optional.of(10000);
+    Optional<Integer> iterations = Optional.of(10000);
     Optional<Double> maxDatanodesRatioToInvolvePerIteration = Optional.of(1d);
     Optional<Long> maxSizeToMovePerIterationInGB = Optional.of(1L);
     Optional<Long> maxSizeEnteringTargetInGB = Optional.of(1L);
     Optional<Long> maxSizeLeavingSourceInGB = Optional.of(1L);
 
-    containerBalancerClient.startContainerBalancer(threshold, idleiterations,
+    containerBalancerClient.startContainerBalancer(threshold, iterations,
         maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB,
         maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB);
     running = containerBalancerClient.getContainerBalancerStatus();
@@ -104,7 +104,7 @@ public void testContainerBalancerCLIOperations() throws Exception {
     assertFalse(running);
 
     // test normally start , and stop it before balance is completed
-    containerBalancerClient.startContainerBalancer(threshold, idleiterations,
+    containerBalancerClient.startContainerBalancer(threshold, iterations,
         maxDatanodesRatioToInvolvePerIteration, maxSizeToMovePerIterationInGB,
         maxSizeEnteringTargetInGB, maxSizeLeavingSourceInGB);
     running = containerBalancerClient.getContainerBalancerStatus();