diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java index ca82a61d134d..b5d3ca2248c1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -41,6 +42,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult; +import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache; import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil; @@ -89,6 +91,7 @@ public class HddsVolume extends StorageVolume { private ContainerController controller; private final AtomicLong committedBytes = new AtomicLong(); // till Open containers become full + private Function gatherContainerUsages = (K) -> 0L; // Mentions the type of volume private final VolumeType type = VolumeType.DATA_VOLUME; @@ -405,6 +408,38 @@ public long getFreeSpaceToSpare(long volumeCapacity) { return getDatanodeConfig().getMinFreeSpace(volumeCapacity); } + @Override + public void setGatherContainerUsages(Function gatherContainerUsages) { + this.gatherContainerUsages = gatherContainerUsages; + } + + @Override + protected long containerUsedSpace() { + return gatherContainerUsages.apply(this); + } + + @Override + public File getContainerDirsPath() { + if (getStorageState() != VolumeState.NORMAL) { + return null; + } + File hddsVolumeRootDir = getHddsRootDir(); + //filtering storage directory + File[] storageDirs = hddsVolumeRootDir.listFiles(File::isDirectory); + if (storageDirs == null) { + LOG.error("IO error for the volume {}, directory not found", hddsVolumeRootDir); + return null; + } + File clusterIDDir = new File(hddsVolumeRootDir, getClusterID()); + if (storageDirs.length == 1 && !clusterIDDir.exists()) { + // If this volume was formatted pre SCM HA, this will be the SCM ID. + // A cluster ID symlink will exist in this case only if this cluster is finalized for SCM HA. + // If the volume was formatted post SCM HA, this will be the cluster ID. + clusterIDDir = storageDirs[0]; + } + return new File(clusterIDDir, Storage.STORAGE_DIR_CURRENT); + } + public void setDbVolume(DbVolume dbVolume) { this.dbVolume = dbVolume; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java index 9616eeeab9ab..ce979cfdb6a1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.fs.SpaceUsageCheckFactory; @@ -273,10 +274,20 @@ public void checkVolumeAsync(StorageVolume volume) { }); } + public void startAllVolume() throws IOException { + for (Map.Entry entry : volumeMap.entrySet()) { + entry.getValue().start(); + } + } + public void refreshAllVolumeUsage() { volumeMap.forEach((k, v) -> v.refreshVolumeUsage()); } + public void setGatherContainerUsages(Function gatherContainerUsages) { + volumeMap.forEach((k, v) -> v.setGatherContainerUsages(gatherContainerUsages)); + } + /** * Acquire Volume Set Read lock. */ diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java index b75de69aea66..3ef4008463a7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java @@ -33,6 +33,8 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -148,7 +150,8 @@ protected StorageVolume(Builder b) throws IOException { if (!b.failedVolume) { StorageLocation location = StorageLocation.parse(volumeRoot); storageDir = new File(location.getUri().getPath(), b.storageDirStr); - SpaceUsageCheckParams checkParams = getSpaceUsageCheckParams(b); + SpaceUsageCheckParams checkParams = getSpaceUsageCheckParams(b, this::getContainerDirsPath); + checkParams.setContainerUsedSpace(this::containerUsedSpace); volumeUsage = Optional.of(new VolumeUsage(checkParams, b.conf)); this.volumeSet = b.volumeSet; this.state = VolumeState.NOT_INITIALIZED; @@ -176,6 +179,20 @@ protected StorageVolume(Builder b) throws IOException { this.storageDirStr = storageDir.getAbsolutePath(); } + protected long containerUsedSpace() { + // container used space applicable only for HddsVolume + return 0; + } + + public File getContainerDirsPath() { + // container dir path applicable only for HddsVolume + return null; + } + + public void setGatherContainerUsages(Function gatherContainerUsages) { + // Operation only for HddsVolume which have container data + } + public void format(String cid) throws IOException { Preconditions.checkNotNull(cid, "clusterID cannot be null while " + "formatting Volume"); @@ -183,6 +200,10 @@ public void format(String cid) throws IOException { initialize(); } + public void start() throws IOException { + volumeUsage.ifPresent(VolumeUsage::start); + } + /** * Initializes the volume. * Creates the Version file if not present, @@ -723,7 +744,8 @@ public String toString() { return getStorageDir().toString(); } - private static SpaceUsageCheckParams getSpaceUsageCheckParams(Builder b) throws IOException { + private static SpaceUsageCheckParams getSpaceUsageCheckParams(Builder b, Supplier exclusionProvider) + throws IOException { File root = new File(b.volumeRootStr); boolean succeeded = root.isDirectory() || root.mkdirs(); @@ -738,6 +760,6 @@ private static SpaceUsageCheckParams getSpaceUsageCheckParams(Builder b) throws usageCheckFactory = SpaceUsageCheckFactory.create(b.conf); } - return usageCheckFactory.paramsFor(root); + return usageCheckFactory.paramsFor(root, exclusionProvider); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java index 0fbfb474b402..86b80afcdbd9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java @@ -106,7 +106,6 @@ public class VolumeUsage { source = new CachingSpaceUsageSource(checkParams); reservedInBytes = getReserved(conf, checkParams.getPath(), source.getCapacity()); Preconditions.assertTrue(reservedInBytes >= 0, reservedInBytes + " < 0"); - start(); // TODO should start only on demand } @VisibleForTesting diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index fffc17db3577..cf1da771a2c5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -46,6 +46,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -195,6 +196,7 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService, this.witnessedContainerMetadataStore = WitnessedContainerMetadataStoreImpl.get(conf); containerSet = ContainerSet.newRwContainerSet(witnessedContainerMetadataStore.getContainerIdsTable(), recoveringContainerTimeout); + volumeSet.setGatherContainerUsages(this::gatherContainerUsages); metadataScanner = null; metrics = ContainerMetrics.create(conf); @@ -503,12 +505,14 @@ public void start(String clusterId) throws IOException { // Do an immediate check of all volumes to ensure datanode health before // proceeding. volumeSet.checkAllVolumes(); + volumeSet.startAllVolume(); metaVolumeSet.checkAllVolumes(); + metaVolumeSet.startAllVolume(); // DB volume set may be null if dedicated DB volumes are not used. if (dbVolumeSet != null) { dbVolumeSet.checkAllVolumes(); + dbVolumeSet.startAllVolume(); } - LOG.info("Attempting to start container services."); startContainerScrub(); @@ -575,6 +579,15 @@ public ContainerSet getContainerSet() { return containerSet; } + public Long gatherContainerUsages(HddsVolume storageVolume) { + AtomicLong usages = new AtomicLong(); + containerSet.getContainerMapIterator().forEachRemaining(e -> { + if (e.getValue().getContainerData().getVolume().getStorageID().equals(storageVolume.getStorageID())) { + usages.addAndGet(e.getValue().getContainerData().getBytesUsed()); + } + }); + return usages.get(); + } /** * Returns container report. * diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java index 1451a14f0f87..3d8399f28301 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java @@ -90,7 +90,6 @@ import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; @@ -145,8 +144,10 @@ public void testContainerCloseActionWhenFull( conf.set(HDDS_DATANODE_DIR_KEY, testDirPath); conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDirPath); DatanodeDetails dd = randomDatanodeDetails(); - MutableVolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, + MutableVolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), "test", conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); + volumeSet.getVolumesList().forEach(e -> e.setState(StorageVolume.VolumeState.NORMAL)); + volumeSet.startAllVolume(); try { UUID scmId = UUID.randomUUID(); @@ -294,7 +295,7 @@ public void testContainerCloseActionWhenVolumeFull( HddsVolume.Builder volumeBuilder = new HddsVolume.Builder(testDirPath).datanodeUuid(dd.getUuidString()) - .conf(conf).usageCheckFactory(MockSpaceUsageCheckFactory.NONE); + .conf(conf).usageCheckFactory(MockSpaceUsageCheckFactory.NONE).clusterID("test"); // state of cluster : available (160) > 100 ,datanode volume // utilisation threshold not yet reached. container creates are successful. AtomicLong usedSpace = new AtomicLong(340); @@ -306,6 +307,8 @@ public void testContainerCloseActionWhenVolumeFull( MutableVolumeSet volumeSet = mock(MutableVolumeSet.class); when(volumeSet.getVolumesList()) .thenReturn(Collections.singletonList(volumeBuilder.build())); + volumeSet.getVolumesList().get(0).setState(StorageVolume.VolumeState.NORMAL); + volumeSet.getVolumesList().get(0).start(); try { UUID scmId = UUID.randomUUID(); ContainerSet containerSet = newContainerSet(); @@ -566,7 +569,7 @@ static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, OzoneConfiguration conf, TokenVerifier tokenVerifier) throws IOException { ContainerSet containerSet = newContainerSet(); - VolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null, + MutableVolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); volumeSet.getVolumesList().stream().forEach(v -> { try { @@ -576,6 +579,7 @@ static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, throw new RuntimeException(e); } }); + volumeSet.startAllVolume(); StateContext context = ContainerTestUtils.getMockContext(dd, conf); ContainerMetrics metrics = ContainerMetrics.create(conf); Map handlers = Maps.newHashMap(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java index 0e90b091fef1..d36191d03dae 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java @@ -48,6 +48,7 @@ import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.container.common.ContainerTestUtils; import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile; import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache; @@ -244,6 +245,7 @@ public void testShutdown() throws Exception { volumeBuilder.usageCheckFactory(factory); HddsVolume volume = volumeBuilder.build(); + volume.start(); assertEquals(initialUsedSpace, savedUsedSpace.get()); assertEquals(expectedUsedSpace, volume.getCurrentUsage().getUsedSpace()); @@ -299,6 +301,7 @@ public void testReportUsedBiggerThanActualUsed() throws IOException { volumeBuilder.usageCheckFactory(factory); HddsVolume volume = volumeBuilder.build(); + volume.start(); SpaceUsageSource usage = volume.getCurrentUsage(); assertEquals(400, usage.getCapacity()); @@ -354,6 +357,7 @@ public void testOverUsedReservedSpace() throws IOException { volumeBuilder.usageCheckFactory(factory); HddsVolume volume = volumeBuilder.build(); + volume.start(); SpaceUsageSource usage = volume.getCurrentUsage(); assertEquals(400, usage.getCapacity()); @@ -381,6 +385,7 @@ public void testOverUsedHddsSpace() throws IOException { volumeBuilder.usageCheckFactory(factory); HddsVolume volume = volumeBuilder.build(); + volume.start(); SpaceUsageSource usage = volume.getCurrentUsage(); assertEquals(400, usage.getCapacity()); @@ -538,6 +543,25 @@ public void testDBDirFailureDetected() throws Exception { volume.shutdown(); } + @Test + public void testGetContainerDirsPath() throws Exception { + HddsVolume volume = volumeBuilder.build(); + volume.format(CLUSTER_ID); + volume.createWorkingDir(CLUSTER_ID, null); + + File expectedPath = new File(new File(volume.getStorageDir(), CLUSTER_ID), Storage.STORAGE_DIR_CURRENT); + assertEquals(expectedPath, volume.getContainerDirsPath()); + + volume.shutdown(); + } + + @Test + public void testGetContainerDirsPathWhenNotFormatted() throws Exception { + HddsVolume volume = volumeBuilder.build(); + assertNull(volume.getContainerDirsPath()); + volume.shutdown(); + } + @Test public void testVolumeUsagesMetrics() throws Exception { // Build a volume with mocked usage, with reserved: 100B, Min free: 10B diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index de5b2331f12d..ad40f25be60f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -99,6 +99,7 @@ private void setup() throws Exception { clusterId, conf, null, StorageVolume.VolumeType.DATA_VOLUME, null); createDbInstancesForTestIfNeeded(volumeSet, clusterId, clusterId, conf); volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy(); + volumeSet.startAllVolume(); } @AfterEach @@ -142,7 +143,7 @@ public void testBuildContainerMap(ContainerTestVersionInfo versionInfo) keyValueContainer.create(volumeSet, volumeChoosingPolicy, clusterId); myVolume = keyValueContainer.getContainerData().getVolume(); - freeBytes = addBlocks(keyValueContainer, 2, 3); + freeBytes = addBlocks(keyValueContainer, 2, 3, 65536); // update our expectation of volume committed space in the map volCommitBytes = commitSpaceMap.get(getVolumeKey(myVolume)).longValue(); @@ -158,6 +159,8 @@ public void testBuildContainerMap(ContainerTestVersionInfo versionInfo) ContainerSet containerset = ozoneContainer.getContainerSet(); assertEquals(numTestContainers, containerset.containerCount()); verifyCommittedSpace(ozoneContainer); + // container usage here, nrOfContainer * blocks * chunksPerBlock * datalen + assertEquals(10 * 2 * 3 * 65536, ozoneContainer.gatherContainerUsages(volumes.get(0))); Set missingContainers = new HashSet<>(); for (int i = 0; i < numTestContainers; i++) { if (i % 2 == 0) { @@ -262,10 +265,9 @@ private void verifyCommittedSpace(OzoneContainer oc) { } private long addBlocks(KeyValueContainer container, - int blocks, int chunksPerBlock) throws Exception { + int blocks, int chunksPerBlock, int datalen) throws Exception { String strBlock = "block"; String strChunk = "-chunkFile"; - int datalen = 65536; long usedBytes = 0; long freeBytes = container.getContainerData().getMaxSize(); diff --git a/hadoop-hdds/docs/content/design/dn-usedspace-calculation.md b/hadoop-hdds/docs/content/design/dn-usedspace-calculation.md new file mode 100644 index 000000000000..0674f5c95bf2 --- /dev/null +++ b/hadoop-hdds/docs/content/design/dn-usedspace-calculation.md @@ -0,0 +1,91 @@ +--- +title: Datanode used space calculation +summary: Describe proposal for optimization in datanode used space calculation. +date: 2025-04-30 +jira: HDDS-12924 +status: Implemented +author: Sumit Agrawal +--- + + +# Abstract +Datanode makes use of DU to identify disk space uses for ozone in the volume. +DU is a time-consuming operation if run over large occupied disk, like when running over disk path having 10s of TB container data. This will be slow operation. + + +## Approach 1: Run DU over non-ozone path + +Challenges: + +- Root path is not known for the device mounted on. So this needs extra configuration. +- Permission problem: du will not count the space if permission is not there on a certain path. + + +Based on the above concern, it's `not feasible` to do du over non-ozone path. + + +## Approach 2: Run DU over meta path only (excluding container dir path) + +Ozone space usages includes summation of: +- Sum of all Container data size as present in memory +- DU over volume path as current (excluding container path) + +`Used space = sum(, )` + +### Limitation: +- Space is not counted as ozone space for below cases: + - Ozone used size for duplicate containers are not counted (ignored during startup for EC case) + - Ozone used size for containers corrupted are not counted, not deleted + - Container path meta files like container yaml, exported rocks db’s sst files, are not counted, which might result in few GB of data in a large cluster. + + These spaces will be added up to non-ozone used space, and especially un-accounted containers need to be cleaned up. + +In future, the container space (due to duplicacy or corruption) needs to be removed based on error logs. + + +### Impact of inaccuracy of Used Space + +1. Used space in reporting (may be few GB as per above) +2. Adjustment between Ozone available space and Reserved available space + +This inaccuracy does not have much impact over solution (as its present in existing) and due to nature of “du” running async and parallel write operation being in progress. + +Approach to fits better in this scenario, can be provided as `OptimizedDU` and keeping previous support as well. + +## Approach 3 : Used space will be only container data + +Hadoop also have similar concept, where used space is only the actual data blocks, and its calculation is simple, +i.e. + +`Used Space = number of blocks * block Size` + +Similar to this, Ozone can also calculate Used space as, + +`Used Space = Sum of all container data size` + +### Impact: + +Space will `not` be counted: +1. Meta space like rocks db, container yaml file and sst files of few KBs per container, which overall result in few GB of space in large volume. +2. duplicate container not yet deleted and corrupted container present in container data path +3. temp path having left over data which is not yet deleted + +These space will be counted in reserved space. And these data are small in nature (except container data). +Container data are corrupted / containers which needs to be removed manually or need fix in code to remove those automatic. + +So considering above impact, this might be one of simple solution providing higher performance. + +# Conclusion + +`Approach 2` `Run DU over meta path only (excluding container dir path)` is preferred over other as +it identify ozone used space more close in accuracy to DU implementation and handle the time issue. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DU.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DU.java index 18d12cb7f219..7acd05badb0c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DU.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DU.java @@ -24,6 +24,7 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.util.Shell; import org.slf4j.Logger; @@ -42,7 +43,7 @@ public class DU extends AbstractSpaceUsageSource { private final DUShell duShell; private final String[] command; private final String commandString; - private final String excludePattern; + private final Supplier exclusionProvider; public DU(File path) { this(path, null); @@ -51,10 +52,19 @@ public DU(File path) { public DU(File path, String excludePattern) { super(path); - this.excludePattern = excludePattern; - command = constructCommand(); + command = constructCommand(getPath(), excludePattern); commandString = String.join(" ", command); duShell = new DUShell(); + exclusionProvider = null; + } + + public DU(Supplier exclusionProvider, File path) { + super(path); + + this.exclusionProvider = exclusionProvider; + duShell = new DUShell(); + command = null; + commandString = null; } @Override @@ -62,7 +72,7 @@ public long getUsedSpace() { return time(duShell::getUsed, LOG); } - private String[] constructCommand() { + private static String[] constructCommand(String path, String excludePattern) { List parts = new LinkedList<>(); parts.add("du"); parts.add("-sk"); @@ -74,7 +84,7 @@ private String[] constructCommand() { } parts.add(excludePattern); } - parts.add(getPath()); + parts.add(path); return parts.toArray(new String[0]); } @@ -96,11 +106,21 @@ public long getUsed() { @Override public String toString() { + if (exclusionProvider != null) { + return String.join(" ", getExecString()) + "\n" + value.get() + "\t" + getPath(); + } return commandString + "\n" + value.get() + "\t" + getPath(); } @Override protected String[] getExecString() { + if (exclusionProvider != null) { + File exclusionFile = exclusionProvider.get(); + if (null == exclusionFile) { + return constructCommand(getPath(), null); + } + return constructCommand(getPath(), exclusionFile.getAbsolutePath()); + } return command; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUOptimized.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUOptimized.java new file mode 100644 index 000000000000..1d90c26c7f41 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUOptimized.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.fs; + +import java.io.File; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Make use of DU class that uses the unix 'du' program to calculate space usage of metadata excluding container data. + * Container data usages is identified from container set. + * + * @see SpaceUsageSource + */ +public class DUOptimized implements SpaceUsageSource { + private static final Logger LOG = LoggerFactory.getLogger(DUOptimized.class); + + private final DU metaPathDU; + private Supplier> containerUsedSpaceProvider; + + public DUOptimized(File path, Supplier exclusionProvider) { + metaPathDU = new DU(exclusionProvider, path); + } + + @Override + public long getUsedSpace() { + long metaPathSize = metaPathDU.getUsedSpace(); + if (null == containerUsedSpaceProvider) { + return metaPathSize; + } + Supplier gatherContainerUsages = containerUsedSpaceProvider.get(); + long containerUsedSpace = gatherContainerUsages.get(); + LOG.info("Disk metaPath du usages {}, container data usages {}", metaPathSize, containerUsedSpace); + return metaPathSize + containerUsedSpace; + } + + @Override + public long getCapacity() { + return metaPathDU.getCapacity(); + } + + @Override + public long getAvailable() { + return metaPathDU.getAvailable(); + } + + public void setContainerUsedSpaceProvider(Supplier> containerUsedSpaceProvider) { + this.containerUsedSpaceProvider = containerUsedSpaceProvider; + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUOptimizedFactory.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUOptimizedFactory.java new file mode 100644 index 000000000000..043d56dd24f6 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUOptimizedFactory.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.fs; + +import java.io.File; +import java.time.Duration; +import java.util.function.Supplier; +import org.apache.hadoop.hdds.conf.ConfigurationSource; + +/** + * Uses DU for all volumes excluding container data. Container data used space is from memory via container set. + * Saves used value in cache file. + */ +public class DUOptimizedFactory implements SpaceUsageCheckFactory { + + private static final String DU_CACHE_FILE = "scmUsed"; + + private DUFactory.Conf conf; + + @Override + public SpaceUsageCheckFactory setConfiguration( + ConfigurationSource configuration) { + conf = configuration.getObject(DUFactory.Conf.class); + return this; + } + + @Override + public SpaceUsageCheckParams paramsFor(File dir) { + return null; + } + + @Override + public SpaceUsageCheckParams paramsFor(File dir, Supplier exclusionProvider) { + Duration refreshPeriod = conf.getRefreshPeriod(); + DUOptimized source = new DUOptimized(dir, exclusionProvider); + SpaceUsagePersistence persistence = new SaveSpaceUsageToFile( + new File(dir, DU_CACHE_FILE), refreshPeriod); + + SpaceUsageCheckParams params = new SpaceUsageCheckParams(dir, source, refreshPeriod, persistence); + source.setContainerUsedSpaceProvider(params::getContainerUsedSpace); + return params; + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java index 0d60620d87d0..62bfca021666 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckFactory.java @@ -21,6 +21,7 @@ import java.io.UncheckedIOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.function.Supplier; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.Config; @@ -53,6 +54,15 @@ public interface SpaceUsageCheckFactory { */ SpaceUsageCheckParams paramsFor(File dir); + /** + * Creates configuration for the HDDS volume rooted at {@code dir} with exclusion path for du. + * + * @throws UncheckedIOException if canonical path for {@code dir} cannot be resolved + */ + default SpaceUsageCheckParams paramsFor(File dir, Supplier exclusionProvider) { + return paramsFor(dir); + } + /** * Updates the factory with global configuration. * @return factory configured with {@code conf} @@ -108,8 +118,8 @@ static SpaceUsageCheckFactory create(ConfigurationSource config) { return instance.setConfiguration(config); } - static DUFactory defaultImplementation() { - return new DUFactory(); + static SpaceUsageCheckFactory defaultImplementation() { + return new DUOptimizedFactory(); } /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckParams.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckParams.java index 8e0128d32065..e20503b00256 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckParams.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/SpaceUsageCheckParams.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.time.Duration; +import java.util.function.Supplier; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; @@ -39,6 +40,7 @@ public class SpaceUsageCheckParams { private final SpaceUsagePersistence persistence; private final String path; private final File dir; + private Supplier containerUsedSpace = () -> 0L; /** * @param refresh The period of refreshing space usage information from @@ -88,4 +90,11 @@ public SpaceUsagePersistence getPersistence() { return persistence; } + public void setContainerUsedSpace(Supplier containerUsedSpace) { + this.containerUsedSpace = containerUsedSpace; + } + + public Supplier getContainerUsedSpace() { + return containerUsedSpace; + } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/fs/TestDUOptimized.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/fs/TestDUOptimized.java new file mode 100644 index 000000000000..e43e5cfb7cdc --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/fs/TestDUOptimized.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.fs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.function.Supplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link DUOptimized}. + */ +class TestDUOptimized { + private DUOptimized duOptimized; + private final DU metaPathDUMock = mock(DU.class); + + @BeforeEach + void setUp() throws Exception { + duOptimized = new DUOptimized(new File("/tmp"), () -> new File("/tmp/exclude")); + + Field field = DUOptimized.class.getDeclaredField("metaPathDU"); + field.setAccessible(true); + field.set(duOptimized, metaPathDUMock); + } + + @Test + void testGetUsedSpaceWithoutContainerProvider() { + when(metaPathDUMock.getUsedSpace()).thenReturn(100L); + assertEquals(100L, duOptimized.getUsedSpace()); + } + + @Test + void testGetUsedSpaceWithContainerProvider() { + when(metaPathDUMock.getUsedSpace()).thenReturn(100L); + Supplier containerUsage = () -> 50L; + duOptimized.setContainerUsedSpaceProvider(() -> containerUsage); + assertEquals(150L, duOptimized.getUsedSpace()); + } + + @Test + void testGetCapacity() { + when(metaPathDUMock.getCapacity()).thenReturn(1000L); + assertEquals(1000L, duOptimized.getCapacity()); + } + + @Test + void testGetAvailable() { + when(metaPathDUMock.getAvailable()).thenReturn(900L); + assertEquals(900L, duOptimized.getAvailable()); + } +} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/fs/TestDUOptimizedFactory.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/fs/TestDUOptimizedFactory.java new file mode 100644 index 000000000000..e6a3e397174c --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/fs/TestDUOptimizedFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.fs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.io.File; +import java.time.Duration; +import java.util.function.Supplier; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link DUOptimizedFactory}. + */ +class TestDUOptimizedFactory { + + @Test + void testParamsFor(@TempDir File dir) { + Duration refresh = Duration.ofMinutes(30); + OzoneConfiguration conf = new OzoneConfiguration(); + + DUFactory.Conf duConf = conf.getObject(DUFactory.Conf.class); + duConf.setRefreshPeriod(refresh); + conf.setFromObject(duConf); + + DUOptimizedFactory factory = new DUOptimizedFactory(); + factory.setConfiguration(conf); + + Supplier exclusionProvider = () -> new File(dir, "exclude"); + SpaceUsageCheckParams params = factory.paramsFor(dir, exclusionProvider); + + assertSame(dir, params.getDir()); + assertEquals(refresh, params.getRefresh()); + assertSame(DUOptimized.class, params.getSource().getClass()); + assertSame(SaveSpaceUsageToFile.class, params.getPersistence().getClass()); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestRefreshVolumeUsageHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestRefreshVolumeUsageHandler.java index 2cd1090251c7..1cc08ae6b6f1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestRefreshVolumeUsageHandler.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestRefreshVolumeUsageHandler.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.fs.DUOptimized; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; @@ -86,6 +87,7 @@ public void test() throws Exception { .getScmNodeManager().getUsageInfo(datanodeDetails) .getScmNodeStat().getScmUsed().get(); + GenericTestUtils.LogCapturer logCapture = GenericTestUtils.LogCapturer.captureLogs(DUOptimized.class); //creating a key to take some storage space try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) { ObjectStore objectStore = client.getObjectStore(); @@ -126,6 +128,11 @@ public void test() throws Exception { //waiting for the new usage info is refreshed GenericTestUtils.waitFor(() -> isUsageInfoRefreshed(cluster, datanodeDetails, currentScmUsed), 500, 5 * 1000); + + // force refresh and verify used space in optimized flow + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine().getContainer().getVolumeSet().getVolumesList().get(0) + .getVolumeUsage().get().refreshNow(); + GenericTestUtils.waitFor(() -> logCapture.getOutput().contains("container data usages 4"), 500, 10 * 1000); } }