diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index a8a73c783887..26c6bcf4ee76 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -70,7 +70,6 @@ import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.common.volume.VolumeUsage; import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner; import org.apache.hadoop.util.Time; import org.apache.ratis.statemachine.StateMachine; @@ -615,7 +614,12 @@ private boolean isVolumeFull(Container container) { .orElse(Boolean.FALSE); if (isOpen) { HddsVolume volume = container.getContainerData().getVolume(); - return VolumeUsage.getUsableSpace(volume.getReport()) <= 0; + StorageLocationReport volumeReport = volume.getReport(); + boolean full = volumeReport.getUsableSpace() <= 0; + if (full) { + LOG.info("Container {} volume is full: {}", container.getContainerData().getContainerID(), volumeReport); + } + return full; } return false; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java index 9fd094c6a6ab..5f539e121b39 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.ozone.container.common.interfaces.StorageLocationReportMXBean; +import org.apache.hadoop.ozone.container.common.volume.VolumeUsage; /** * Storage location stats of datanodes that provide back store for containers. @@ -54,6 +55,10 @@ private StorageLocationReport(Builder builder) { this.storageLocation = builder.storageLocation; } + public long getUsableSpace() { + return VolumeUsage.getUsableSpace(this); + } + @Override public String getId() { return id; @@ -227,6 +232,27 @@ public static StorageLocationReport getFromProtobuf(StorageReportProto report) return builder.build(); } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(128) + .append("{") + .append(" id=").append(id) + .append(" dir=").append(storageLocation) + .append(" type=").append(storageType); + + if (failed) { + sb.append(" failed"); + } else { + sb.append(" capacity=").append(capacity) + .append(" used=").append(scmUsed) + .append(" available=").append(remaining) + .append(" minFree=").append(freeSpaceToSpare) + .append(" committed=").append(committed); + } + + return sb.append(" }").toString(); + } + /** * Returns StorageLocation.Builder instance. * diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java index ef50e73cb314..6769a8eeaee6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java @@ -17,8 +17,8 @@ package org.apache.hadoop.ozone.container.common.volume; -import java.util.HashMap; -import java.util.Map; +import java.util.LinkedList; +import java.util.List; import java.util.function.Predicate; import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; @@ -29,24 +29,23 @@ public class AvailableSpaceFilter implements Predicate { private final long requiredSpace; - private final Map fullVolumes = - new HashMap<>(); + private final List fullVolumes = new LinkedList<>(); private long mostAvailableSpace = Long.MIN_VALUE; - public AvailableSpaceFilter(long requiredSpace) { + AvailableSpaceFilter(long requiredSpace) { this.requiredSpace = requiredSpace; } @Override public boolean test(HddsVolume vol) { StorageLocationReport report = vol.getReport(); - long available = VolumeUsage.getUsableSpace(report); + long available = report.getUsableSpace(); boolean hasEnoughSpace = available > requiredSpace; mostAvailableSpace = Math.max(available, mostAvailableSpace); if (!hasEnoughSpace) { - fullVolumes.put(vol, new AvailableSpace(report)); + fullVolumes.add(report); } return hasEnoughSpace; @@ -65,18 +64,4 @@ public String toString() { return "required space: " + requiredSpace + ", volumes: " + fullVolumes; } - - private static class AvailableSpace { - private final StorageLocationReport report; - - AvailableSpace(StorageLocationReport report) { - this.report = report; - } - - @Override - public String toString() { - return "free: " + report.getRemaining() + - ", committed: " + report.getCommitted(); - } - } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java index 0f188e53b137..a1f13998b859 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/DbVolume.java @@ -55,13 +55,12 @@ protected DbVolume(Builder b) throws IOException { super(b); this.hddsDbStorePathMap = new HashMap<>(); - if (!b.getFailedVolume()) { - LOG.info("Creating DbVolume: {} of storage type: {}, {}", - getStorageDir(), b.getStorageType(), - getCurrentUsage()); + if (!b.getFailedVolume()) { initialize(); } + + LOG.info("DbVolume: {}", getReport()); } @Override diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java index a81bde59b4fc..55bd55021093 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java @@ -129,11 +129,6 @@ private HddsVolume(Builder b) throws IOException { this.volumeInfoMetrics = new VolumeInfoMetrics(b.getVolumeRootStr(), this); - LOG.info("Creating HddsVolume: {} of storage type: {}, {}", - getStorageDir(), - b.getStorageType(), - getCurrentUsage()); - initialize(); } else { // Builder is called with failedVolume set, so create a failed volume @@ -142,6 +137,8 @@ private HddsVolume(Builder b) throws IOException { volumeIOStats = null; volumeInfoMetrics = new VolumeInfoMetrics(b.getVolumeRootStr(), this); } + + LOG.info("HddsVolume: {}", getReport()); } @Override diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java index 9269619da9e2..fe0fa896921b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java @@ -123,6 +123,8 @@ public enum VolumeState { private final StorageType storageType; private final String volumeRoot; private final File storageDir; + /** This is the raw storage dir location, saved for logging, to avoid repeated filesystem lookup. */ + private final String storageDirStr; private String workingDirName; private File tmpDir; private File diskCheckDir; @@ -173,6 +175,7 @@ protected StorageVolume(Builder b) throws IOException { this.conf = null; this.dnConf = null; } + this.storageDirStr = storageDir.getAbsolutePath(); } public void format(String cid) throws IOException { @@ -459,7 +462,7 @@ protected StorageLocationReport.Builder reportBuilder() { StorageLocationReport.Builder builder = StorageLocationReport.newBuilder() .setFailed(isFailed()) .setId(getStorageID()) - .setStorageLocation(volumeRoot) + .setStorageLocation(storageDirStr) .setStorageType(storageType); if (!builder.isFailed()) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java index 8c44d0d07816..dc53ea8c73a5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; -import org.apache.hadoop.ozone.container.common.volume.AvailableSpaceFilter; +import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.replication.AbstractReplicationTask.Status; import org.slf4j.Logger; @@ -83,10 +83,10 @@ public void replicate(ReplicationTask task) { targetVolume = containerImporter.chooseNextVolume(); // Increment committed bytes and verify if it doesn't cross the space left. targetVolume.incCommittedBytes(containerSize * 2); + StorageLocationReport volumeReport = targetVolume.getReport(); // Already committed bytes increased above, so required space is not required here in AvailableSpaceFilter - AvailableSpaceFilter filter = new AvailableSpaceFilter(0); - if (!filter.test(targetVolume)) { - LOG.warn("Container {} replication was unsuccessful, due to no space left", containerID); + if (volumeReport.getUsableSpace() <= 0) { + LOG.warn("Container {} replication was unsuccessful, no space left on volume {}", containerID, volumeReport); task.setStatus(Status.FAILED); return; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java index e76a44e680de..5224498e7274 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.volume.AvailableSpaceFilter; +import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.util.DiskChecker; import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller; @@ -89,11 +89,11 @@ public void onNext(SendContainerRequest req) { volume = importer.chooseNextVolume(); // Increment committed bytes and verify if it doesn't cross the space left. volume.incCommittedBytes(importer.getDefaultContainerSize() * 2); + StorageLocationReport volumeReport = volume.getReport(); // Already committed bytes increased above, so required space is not required here in AvailableSpaceFilter - AvailableSpaceFilter filter = new AvailableSpaceFilter(0); - if (!filter.test(volume)) { + if (volumeReport.getUsableSpace() <= 0) { volume.incCommittedBytes(-importer.getDefaultContainerSize() * 2); - LOG.warn("Container {} import was unsuccessful, due to no space left", containerId); + LOG.warn("Container {} import was unsuccessful, no space left on volume {}", containerId, volumeReport); volume = null; throw new DiskChecker.DiskOutOfSpaceException("No more available volumes"); }