HDDS-12721. Log more details about volumes

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java

@@ -70,7 +70,6 @@
 import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;
 import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
 import org.apache.hadoop.util.Time;
 import org.apache.ratis.statemachine.StateMachine;
@@ -615,7 +614,12 @@ private boolean isVolumeFull(Container container) {
         .orElse(Boolean.FALSE);
     if (isOpen) {
       HddsVolume volume = container.getContainerData().getVolume();
-      return VolumeUsage.getUsableSpace(volume.getReport()) <= 0;
+      StorageLocationReport volumeReport = volume.getReport();
+      boolean full = volumeReport.getUsableSpace() <= 0;
+      if (full) {
+        LOG.info("Container {} volume is full: {}", container.getContainerData().getContainerID(), volumeReport);
+      }
+      return full;
     }
     return false;
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
 import org.apache.hadoop.ozone.container.common.interfaces.StorageLocationReportMXBean;
+import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;
 
 /**
  * Storage location stats of datanodes that provide back store for containers.
@@ -54,6 +55,10 @@ private StorageLocationReport(Builder builder) {
     this.storageLocation = builder.storageLocation;
   }
 
+  public long getUsableSpace() {
+    return VolumeUsage.getUsableSpace(this);
+  }
+
   @Override
   public String getId() {
     return id;
@@ -227,6 +232,27 @@ public static StorageLocationReport getFromProtobuf(StorageReportProto report)
     return builder.build();
   }
 
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder(128)
+        .append("{")
+        .append(" id=").append(id)
+        .append(" dir=").append(storageLocation)
+        .append(" type=").append(storageType);
+
+    if (failed) {
+      sb.append(" failed");
+    } else {
+      sb.append(" capacity=").append(capacity)
+          .append(" used=").append(scmUsed)
+          .append(" available=").append(remaining)
+          .append(" minFree=").append(freeSpaceToSpare)
+          .append(" committed=").append(committed);
+    }
+
+    return sb.append(" }").toString();
+  }
+
   /**
    * Returns StorageLocation.Builder instance.
    *

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java

@@ -17,8 +17,8 @@
 
 package org.apache.hadoop.ozone.container.common.volume;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.function.Predicate;
 import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
 
@@ -29,24 +29,23 @@
 public class AvailableSpaceFilter implements Predicate<HddsVolume> {
 
   private final long requiredSpace;
-  private final Map<HddsVolume, AvailableSpace> fullVolumes =
-      new HashMap<>();
+  private final List<StorageLocationReport> fullVolumes = new LinkedList<>();
   private long mostAvailableSpace = Long.MIN_VALUE;
 
-  public AvailableSpaceFilter(long requiredSpace) {
+  AvailableSpaceFilter(long requiredSpace) {
     this.requiredSpace = requiredSpace;
   }
 
   @Override
   public boolean test(HddsVolume vol) {
     StorageLocationReport report = vol.getReport();
-    long available = VolumeUsage.getUsableSpace(report);
+    long available = report.getUsableSpace();
     boolean hasEnoughSpace = available > requiredSpace;
 
     mostAvailableSpace = Math.max(available, mostAvailableSpace);
 
     if (!hasEnoughSpace) {
-      fullVolumes.put(vol, new AvailableSpace(report));
+      fullVolumes.add(report);
     }
 
     return hasEnoughSpace;
@@ -65,18 +64,4 @@ public String toString() {
     return "required space: " + requiredSpace +
         ", volumes: " + fullVolumes;
   }
-
-  private static class AvailableSpace {
-    private final StorageLocationReport report;
-
-    AvailableSpace(StorageLocationReport report) {
-      this.report = report;
-    }
-
-    @Override
-    public String toString() {
-      return "free: " + report.getRemaining() +
-          ", committed: " + report.getCommitted();
-    }
-  }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java

@@ -55,13 +55,12 @@ protected DbVolume(Builder b) throws IOException {
     super(b);
 
     this.hddsDbStorePathMap = new HashMap<>();
-    if (!b.getFailedVolume()) {
-      LOG.info("Creating DbVolume: {} of storage type: {}, {}",
-              getStorageDir(), b.getStorageType(),
-              getCurrentUsage());
 
+    if (!b.getFailedVolume()) {
       initialize();
     }
+
+    LOG.info("DbVolume: {}", getReport());
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java

@@ -129,11 +129,6 @@ private HddsVolume(Builder b) throws IOException {
       this.volumeInfoMetrics =
           new VolumeInfoMetrics(b.getVolumeRootStr(), this);
 
-      LOG.info("Creating HddsVolume: {} of storage type: {}, {}",
-          getStorageDir(),
-          b.getStorageType(),
-          getCurrentUsage());
-
       initialize();
     } else {
       // Builder is called with failedVolume set, so create a failed volume
@@ -142,6 +137,8 @@ private HddsVolume(Builder b) throws IOException {
       volumeIOStats = null;
       volumeInfoMetrics = new VolumeInfoMetrics(b.getVolumeRootStr(), this);
     }
+
+    LOG.info("HddsVolume: {}", getReport());
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java

@@ -123,6 +123,8 @@ public enum VolumeState {
   private final StorageType storageType;
   private final String volumeRoot;
   private final File storageDir;
+  /** This is the raw storage dir location, saved for logging, to avoid repeated filesystem lookup. */
+  private final String storageDirStr;
   private String workingDirName;
   private File tmpDir;
   private File diskCheckDir;
@@ -173,6 +175,7 @@ protected StorageVolume(Builder<?> b) throws IOException {
       this.conf = null;
       this.dnConf = null;
     }
+    this.storageDirStr = storageDir.getAbsolutePath();
   }
 
   public void format(String cid) throws IOException {
@@ -459,7 +462,7 @@ protected StorageLocationReport.Builder reportBuilder() {
     StorageLocationReport.Builder builder = StorageLocationReport.newBuilder()
         .setFailed(isFailed())
         .setId(getStorageID())
-        .setStorageLocation(volumeRoot)
+        .setStorageLocation(storageDirStr)
         .setStorageType(storageType);
 
     if (!builder.isFailed()) {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java

@@ -26,7 +26,7 @@
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.volume.AvailableSpaceFilter;
+import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status;
 import org.slf4j.Logger;
@@ -83,10 +83,10 @@ public void replicate(ReplicationTask task) {
       targetVolume = containerImporter.chooseNextVolume();
       // Increment committed bytes and verify if it doesn't cross the space left.
       targetVolume.incCommittedBytes(containerSize * 2);
+      StorageLocationReport volumeReport = targetVolume.getReport();
       // Already committed bytes increased above, so required space is not required here in AvailableSpaceFilter
-      AvailableSpaceFilter filter = new AvailableSpaceFilter(0);
-      if (!filter.test(targetVolume)) {
-        LOG.warn("Container {} replication was unsuccessful, due to no space left", containerID);
+      if (volumeReport.getUsableSpace() <= 0) {
+        LOG.warn("Container {} replication was unsuccessful, no space left on volume {}", containerID, volumeReport);
         task.setStatus(Status.FAILED);
         return;
       }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java

@@ -29,7 +29,7 @@
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.volume.AvailableSpaceFilter;
+import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.util.DiskChecker;
 import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller;
@@ -89,11 +89,11 @@ public void onNext(SendContainerRequest req) {
         volume = importer.chooseNextVolume();
         // Increment committed bytes and verify if it doesn't cross the space left.
         volume.incCommittedBytes(importer.getDefaultContainerSize() * 2);
+        StorageLocationReport volumeReport = volume.getReport();
         // Already committed bytes increased above, so required space is not required here in AvailableSpaceFilter
-        AvailableSpaceFilter filter = new AvailableSpaceFilter(0);
-        if (!filter.test(volume)) {
+        if (volumeReport.getUsableSpace() <= 0) {
           volume.incCommittedBytes(-importer.getDefaultContainerSize() * 2);
-          LOG.warn("Container {} import was unsuccessful, due to no space left", containerId);
+          LOG.warn("Container {} import was unsuccessful, no space left on volume {}", containerId, volumeReport);
           volume = null;
           throw new DiskChecker.DiskOutOfSpaceException("No more available volumes");
         }