HDDS-11367. Improve ozone balancing status command output

hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto

@@ -627,19 +627,20 @@ message ContainerBalancerStatusInfo {
 message ContainerBalancerTaskIterationStatusInfo {
   optional int32 iterationNumber = 1;
   optional string iterationResult = 2;
-  optional int64 sizeScheduledForMoveGB = 3;
-  optional int64 dataSizeMovedGB = 4;
+  optional int64 sizeScheduledForMove = 3;
+  optional int64 dataSizeMoved = 4;
   optional int64 containerMovesScheduled = 5;
   optional int64 containerMovesCompleted = 6;
   optional int64 containerMovesFailed = 7;
   optional int64 containerMovesTimeout = 8;
-  repeated NodeTransferInfo sizeEnteringNodesGB = 9;
-  repeated NodeTransferInfo sizeLeavingNodesGB = 10;
+  repeated NodeTransferInfo sizeEnteringNodes = 9;
+  repeated NodeTransferInfo sizeLeavingNodes = 10;
+  optional int64 iterationDuration = 11;
 }
 
 message NodeTransferInfo {
   optional string uuid = 1;
-  optional int64 dataVolumeGB = 2;
+  optional int64 dataVolume = 2;
 }
 
 message DecommissionScmRequestProto {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerMetrics.java

@@ -40,6 +40,9 @@ public final class ContainerBalancerMetrics {
       " in the latest iteration.")
   private MutableCounterLong dataSizeMovedGBInLatestIteration;
 
+  @Metric(about = "Amount of bytes that Container Balancer moved in the latest iteration.")
+  private MutableCounterLong dataSizeMovedBytesInLatestIteration;
+
   @Metric(about = "Number of completed container moves performed by " +
       "Container Balancer in the latest iteration.")
   private MutableCounterLong numContainerMovesCompletedInLatestIteration;
@@ -131,14 +134,16 @@ void incrementNumContainerMovesScheduledInLatestIteration(long valueToAdd) {
     this.numContainerMovesScheduledInLatestIteration.incr(valueToAdd);
   }
 
+  /**
+   * Reset number of containers scheduled to move in last iteration.
+   */
   public void resetNumContainerMovesScheduledInLatestIteration() {
     numContainerMovesScheduledInLatestIteration.incr(
             -getNumContainerMovesScheduledInLatestIteration());
   }
 
   /**
-   * Gets the amount of data moved by Container Balancer in the latest
-   * iteration.
+   * Gets the amount of data moved by Container Balancer in the latest iteration.
    * @return size in GB
    */
   public long getDataSizeMovedGBInLatestIteration() {
@@ -154,6 +159,29 @@ public void resetDataSizeMovedGBInLatestIteration() {
         -getDataSizeMovedGBInLatestIteration());
   }
 
+  /**
+   * Gets the amount of data moved by Container Balancer in the latest iteration.
+   * @return size in bytes
+   */
+  public long getDataSizeMovedInLatestIteration() {
+    return dataSizeMovedBytesInLatestIteration.value();
+  }
+
+  /**
+   * Increment data size moved in last iteration.
+   * @param bytes bytes to add
+   */
+  public void incrementDataSizeMovedInLatestIteration(long bytes) {
+    this.dataSizeMovedBytesInLatestIteration.incr(bytes);
+  }
+
+  /**
+   * Reset data size moved in last iteration.
+   */
+  public void resetDataSizeMovedInLatestIteration() {
+    dataSizeMovedBytesInLatestIteration.incr(-getDataSizeMovedInLatestIteration());
+  }
+
   /**
    * Gets the number of container moves performed by Container Balancer in the
    * latest iteration.
@@ -163,11 +191,6 @@ public long getNumContainerMovesCompletedInLatestIteration() {
     return numContainerMovesCompletedInLatestIteration.value();
   }
 
-  public void incrementNumContainerMovesCompletedInLatestIteration(
-      long valueToAdd) {
-    this.numContainerMovesCompletedInLatestIteration.incr(valueToAdd);
-  }
-
   public void incrementCurrentIterationContainerMoveMetric(
       MoveManager.MoveResult result, long valueToAdd) {
     if (result == null) {
@@ -204,9 +227,11 @@ public void incrementCurrentIterationContainerMoveMetric(
     }
   }
 
+  /**
+   * Moved containers in last iteration.
+   */
   public void resetNumContainerMovesCompletedInLatestIteration() {
-    numContainerMovesCompletedInLatestIteration.incr(
-        -getNumContainerMovesCompletedInLatestIteration());
+    numContainerMovesCompletedInLatestIteration.incr(-getNumContainerMovesCompletedInLatestIteration());
   }
 
   /**
@@ -218,14 +243,19 @@ public long getNumContainerMovesTimeoutInLatestIteration() {
     return numContainerMovesTimeoutInLatestIteration.value();
   }
 
+  /**
+   * Increment number timeouted container moves.
+   */
   public void incrementNumContainerMovesTimeoutInLatestIteration(
       long valueToAdd) {
     this.numContainerMovesTimeoutInLatestIteration.incr(valueToAdd);
   }
 
+  /**
+   * Reset number timeouted container moves.
+   */
   public void resetNumContainerMovesTimeoutInLatestIteration() {
-    numContainerMovesTimeoutInLatestIteration.incr(
-        -getNumContainerMovesTimeoutInLatestIteration());
+    numContainerMovesTimeoutInLatestIteration.incr(-getNumContainerMovesTimeoutInLatestIteration());
   }
 
   /**

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerStatusInfo.java

@@ -19,9 +19,11 @@
 package org.apache.hadoop.hdds.scm.container.balancer;
 
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Info about balancer status.
@@ -51,4 +53,21 @@ public HddsProtos.ContainerBalancerConfigurationProto  getConfiguration() {
   public List<ContainerBalancerTaskIterationStatusInfo> getIterationsStatusInfo() {
     return iterationsStatusInfo;
   }
+
+  /**
+   * Map to proto.
+   * @return proto representation
+   */
+  public StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfo toProto() {
+    return StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfo
+        .newBuilder()
+        .setStartedAt(getStartedAt().toEpochSecond())
+        .setConfiguration(getConfiguration())
+        .addAllIterationsStatusInfo(
+            getIterationsStatusInfo()
+                .stream()
+                .map(ContainerBalancerTaskIterationStatusInfo::toProto)
+                .collect(Collectors.toList())
+        ).build();
+  }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java

@@ -43,6 +43,7 @@
 
 import java.io.IOException;
 import java.time.Duration;
+import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -60,8 +61,10 @@
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
+import static java.time.OffsetDateTime.now;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL_DEFAULT;
+import static org.apache.hadoop.util.StringUtils.byteDesc;
 
 /**
  * Container balancer task performs move of containers between over- and
@@ -118,7 +121,7 @@ public class ContainerBalancerTask implements Runnable {
   private int nextIterationIndex;
   private boolean delayStart;
   private List<ContainerBalancerTaskIterationStatusInfo> iterationsStatistic;
-
+  private OffsetDateTime currentIterationStarted;
   /**
    * Constructs ContainerBalancerTask with the specified arguments.
    *
@@ -215,6 +218,7 @@ private void balance() {
     // leader change or restart
     int i = nextIterationIndex;
     for (; i < iterations && isBalancerRunning(); i++) {
+      currentIterationStarted = now();
       // reset some variables and metrics for this iteration
       resetState();
       if (config.getTriggerDuEnable()) {
@@ -262,7 +266,7 @@ private void balance() {
       }
 
       IterationResult iR = doIteration();
-      saveIterationStatistic(i, iR);
+      saveIterationStatistic(i + 1, iR);
       metrics.incrementNumIterations(1);
 
       LOG.info("Result of this iteration of Container Balancer: {}", iR);
@@ -307,53 +311,53 @@ private void balance() {
   }
 
   private void saveIterationStatistic(Integer iterationNumber, IterationResult iR) {
+    long iterationDuration = now().toEpochSecond() - currentIterationStarted.toEpochSecond();
     ContainerBalancerTaskIterationStatusInfo iterationStatistic = new ContainerBalancerTaskIterationStatusInfo(
-            iterationNumber,
-            iR.name(),
-            getSizeScheduledForMoveInLatestIteration() / OzoneConsts.GB,
-            metrics.getDataSizeMovedGBInLatestIteration(),
-            metrics.getNumContainerMovesScheduledInLatestIteration(),
-            metrics.getNumContainerMovesCompletedInLatestIteration(),
-            metrics.getNumContainerMovesFailedInLatestIteration(),
-            metrics.getNumContainerMovesTimeoutInLatestIteration(),
-            findTargetStrategy.getSizeEnteringNodes()
-                    .entrySet()
-                    .stream()
-                    .filter(Objects::nonNull)
-                    .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0)
-                    .collect(
-                            Collectors.toMap(
-                                    entry -> entry.getKey().getUuid(),
-                                    entry -> entry.getValue() / OzoneConsts.GB
-                            )
-                    ),
-            findSourceStrategy.getSizeLeavingNodes()
-                    .entrySet()
-                    .stream()
-                    .filter(Objects::nonNull)
-                    .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0)
-                    .collect(
-                            Collectors.toMap(
-                                    entry -> entry.getKey().getUuid(),
-                                    entry -> entry.getValue() / OzoneConsts.GB
-                            )
-                    )
+        iterationNumber,
+        iR.name(),
+        iterationDuration,
+        getSizeScheduledForMoveInLatestIteration(),
+        metrics.getDataSizeMovedInLatestIteration(),
+        metrics.getNumContainerMovesScheduledInLatestIteration(),
+        metrics.getNumContainerMovesCompletedInLatestIteration(),
+        metrics.getNumContainerMovesFailedInLatestIteration(),
+        metrics.getNumContainerMovesTimeoutInLatestIteration(),
+        findTargetStrategy.getSizeEnteringNodes()
+            .entrySet()
+            .stream()
+            .filter(Objects::nonNull)
+            .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0)
+            .collect(
+                Collectors.toMap(
+                    entry -> entry.getKey().getUuid(),
+                    Map.Entry::getValue
+                )
+            ),
+        findSourceStrategy.getSizeLeavingNodes()
+            .entrySet()
+            .stream()
+            .filter(Objects::nonNull)
+            .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0)
+            .collect(
+                Collectors.toMap(
+                    entry -> entry.getKey().getUuid(),
+                    Map.Entry::getValue
+                )
+            )
     );
     iterationsStatistic.add(iterationStatistic);
   }
 
   public List<ContainerBalancerTaskIterationStatusInfo> getCurrentIterationsStatistic() {
-
-    int lastIterationNumber = iterationsStatistic.stream()
-        .mapToInt(ContainerBalancerTaskIterationStatusInfo::getIterationNumber)
-        .max()
-        .orElse(0);
-
+    int lastIterationNumber = iterationsStatistic.isEmpty() ? 0
+        : iterationsStatistic.get(iterationsStatistic.size() - 1).getIterationNumber();
+    long iterationDuration = getCurrentIterationDuration();
     ContainerBalancerTaskIterationStatusInfo currentIterationStatistic = new ContainerBalancerTaskIterationStatusInfo(
-        lastIterationNumber + 1,
+        lastIterationNumber,
         null,
-        getSizeScheduledForMoveInLatestIteration() / OzoneConsts.GB,
-        sizeActuallyMovedInLatestIteration / OzoneConsts.GB,
+        iterationDuration,
+        getSizeScheduledForMoveInLatestIteration(),
+        sizeActuallyMovedInLatestIteration,
         metrics.getNumContainerMovesScheduledInLatestIteration(),
         metrics.getNumContainerMovesCompletedInLatestIteration(),
         metrics.getNumContainerMovesFailedInLatestIteration(),
@@ -365,7 +369,7 @@ public List<ContainerBalancerTaskIterationStatusInfo> getCurrentIterationsStatis
             .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0)
             .collect(Collectors.toMap(
                     entry -> entry.getKey().getUuid(),
-                    entry -> entry.getValue() / OzoneConsts.GB
+                    Map.Entry::getValue
                 )
             ),
         findSourceStrategy.getSizeLeavingNodes()
@@ -376,7 +380,7 @@ public List<ContainerBalancerTaskIterationStatusInfo> getCurrentIterationsStatis
             .collect(
                 Collectors.toMap(
                     entry -> entry.getKey().getUuid(),
-                    entry -> entry.getValue() / OzoneConsts.GB
+                    Map.Entry::getValue
                 )
             )
     );
@@ -385,6 +389,14 @@ public List<ContainerBalancerTaskIterationStatusInfo> getCurrentIterationsStatis
     return resultList;
   }
 
+  private long getCurrentIterationDuration() {
+    if (currentIterationStarted == null) {
+      return -1L;
+    } else {
+      return now().toEpochSecond() - currentIterationStarted.toEpochSecond();
+    }
+  }
+
   /**
    * Logs the reason for stop and save configuration and stop the task.
    * 
@@ -708,26 +720,28 @@ private void checkIterationMoveResults() {
       }
     }
 
-    countDatanodesInvolvedPerIteration =
-        selectedSources.size() + selectedTargets.size();
-    metrics.incrementNumDatanodesInvolvedInLatestIteration(
-        countDatanodesInvolvedPerIteration);
-    metrics.incrementNumContainerMovesScheduled(
-        metrics.getNumContainerMovesScheduledInLatestIteration());
-    metrics.incrementNumContainerMovesCompleted(
-        metrics.getNumContainerMovesCompletedInLatestIteration());
-    metrics.incrementNumContainerMovesTimeout(
-        metrics.getNumContainerMovesTimeoutInLatestIteration());
-    metrics.incrementDataSizeMovedGBInLatestIteration(
-        sizeActuallyMovedInLatestIteration / OzoneConsts.GB);
-    metrics.incrementDataSizeMovedGB(
-        metrics.getDataSizeMovedGBInLatestIteration());
-    metrics.incrementNumContainerMovesFailed(
-        metrics.getNumContainerMovesFailedInLatestIteration());
+    countDatanodesInvolvedPerIteration = selectedSources.size() + selectedTargets.size();
+
+    metrics.incrementNumDatanodesInvolvedInLatestIteration(countDatanodesInvolvedPerIteration);
+
+    metrics.incrementNumContainerMovesScheduled(metrics.getNumContainerMovesScheduledInLatestIteration());
+
+    metrics.incrementNumContainerMovesCompleted(metrics.getNumContainerMovesCompletedInLatestIteration());
+
+    metrics.incrementNumContainerMovesTimeout(metrics.getNumContainerMovesTimeoutInLatestIteration());
+
+    metrics.incrementDataSizeMovedGBInLatestIteration(sizeActuallyMovedInLatestIteration / OzoneConsts.GB);
+
+    metrics.incrementDataSizeMovedInLatestIteration(sizeActuallyMovedInLatestIteration);
+
+    metrics.incrementDataSizeMovedGB(metrics.getDataSizeMovedGBInLatestIteration());
+
+    metrics.incrementNumContainerMovesFailed(metrics.getNumContainerMovesFailedInLatestIteration());
+
     LOG.info("Iteration Summary. Number of Datanodes involved: {}. Size " +
             "moved: {} ({} Bytes). Number of Container moves completed: {}.",
         countDatanodesInvolvedPerIteration,
-        StringUtils.byteDesc(sizeActuallyMovedInLatestIteration),
+        byteDesc(sizeActuallyMovedInLatestIteration),
         sizeActuallyMovedInLatestIteration,
         metrics.getNumContainerMovesCompletedInLatestIteration());
   }
@@ -1146,6 +1160,7 @@ private void resetState() {
     this.sizeScheduledForMoveInLatestIteration = 0;
     this.sizeActuallyMovedInLatestIteration = 0;
     metrics.resetDataSizeMovedGBInLatestIteration();
+    metrics.resetDataSizeMovedInLatestIteration();
     metrics.resetNumContainerMovesScheduledInLatestIteration();
     metrics.resetNumContainerMovesCompletedInLatestIteration();
     metrics.resetNumContainerMovesTimeoutInLatestIteration();