Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;

/**
* Interface for Container Operations.
Expand Down Expand Up @@ -102,7 +103,7 @@ public Throwable getException() {
* @throws StorageContainerException
*/
void create(VolumeSet volumeSet, VolumeChoosingPolicy volumeChoosingPolicy,
String scmId) throws StorageContainerException;
MinFreeSpaceCalculator minFreeSpaceCalculator, String scmId) throws StorageContainerException;

/**
* Deletes the container.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.List;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;

/**
* This interface specifies the policy for choosing volumes to store replicas.
Expand All @@ -37,9 +38,11 @@ public interface VolumeChoosingPolicy {
* @param volumes - a list of available volumes.
* @param maxContainerSize - the maximum size of the container for which a
* volume is sought.
* @param freeSpaceCalculator - calculator for determining minimum free space
* @return the chosen volume.
* @throws IOException when disks are unavailable or are full.
*/
HddsVolume chooseVolume(List<HddsVolume> volumes, long maxContainerSize)
HddsVolume chooseVolume(List<HddsVolume> volumes, long maxContainerSize,
MinFreeSpaceCalculator freeSpaceCalculator)
throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Map;
import java.util.function.Predicate;
import org.apache.hadoop.hdds.fs.SpaceUsageSource;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;

/**
* Filter for selecting volumes with enough space for a new container.
Expand All @@ -32,9 +33,12 @@ public class AvailableSpaceFilter implements Predicate<HddsVolume> {
private final Map<HddsVolume, AvailableSpace> fullVolumes =
new HashMap<>();
private long mostAvailableSpace = Long.MIN_VALUE;
private final MinFreeSpaceCalculator freeSpaceCalculator;

AvailableSpaceFilter(long requiredSpace) {
AvailableSpaceFilter(long requiredSpace,
MinFreeSpaceCalculator freeSpaceCalculator) {
this.requiredSpace = requiredSpace;
this.freeSpaceCalculator = freeSpaceCalculator;
}

@Override
Expand All @@ -44,8 +48,7 @@ public boolean test(HddsVolume vol) {
long free = usage.getAvailable();
long committed = vol.getCommittedBytes();
long available = free - committed;
long volumeFreeSpaceToSpare =
new VolumeUsage.MinFreeSpaceCalculator(vol.getConf()).get(volumeCapacity);
long volumeFreeSpaceToSpare = freeSpaceCalculator.get(volumeCapacity);
boolean hasEnoughSpace = VolumeUsage.hasVolumeEnoughSpace(free, committed,
requiredSpace, volumeFreeSpaceToSpare);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.Random;
import java.util.stream.Collectors;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -49,14 +50,16 @@ public class CapacityVolumeChoosingPolicy implements VolumeChoosingPolicy {

@Override
public HddsVolume chooseVolume(List<HddsVolume> volumes,
long maxContainerSize) throws IOException {
long maxContainerSize,
MinFreeSpaceCalculator freeSpaceCalculator) throws IOException {

// No volumes available to choose from
if (volumes.isEmpty()) {
throw new DiskOutOfSpaceException("No more available volumes");
}

AvailableSpaceFilter filter = new AvailableSpaceFilter(maxContainerSize);
AvailableSpaceFilter filter = new AvailableSpaceFilter(maxContainerSize,
freeSpaceCalculator);

List<HddsVolume> volumesWithEnoughSpace = volumes.stream()
.filter(filter)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -42,14 +43,16 @@ public class RoundRobinVolumeChoosingPolicy implements VolumeChoosingPolicy {

@Override
public HddsVolume chooseVolume(List<HddsVolume> volumes,
long maxContainerSize) throws IOException {
long maxContainerSize,
MinFreeSpaceCalculator freeSpaceCalculator) throws IOException {

// No volumes available to choose from
if (volumes.isEmpty()) {
throw new DiskOutOfSpaceException("No more available volumes");
}

AvailableSpaceFilter filter = new AvailableSpaceFilter(maxContainerSize);
AvailableSpaceFilter filter = new AvailableSpaceFilter(maxContainerSize,
freeSpaceCalculator);

// since volumes could've been removed because of the failure
// make sure we are not out of bounds
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
Expand Down Expand Up @@ -138,8 +139,8 @@ public void setCheckChunksFilePath(boolean bCheckChunksDirFilePath) {
}

@Override
public void create(VolumeSet volumeSet, VolumeChoosingPolicy
volumeChoosingPolicy, String clusterId) throws StorageContainerException {
public void create(VolumeSet volumeSet, VolumeChoosingPolicy volumeChoosingPolicy,
MinFreeSpaceCalculator minFreeSpaceCalculator, String clusterId) throws StorageContainerException {
Preconditions.checkNotNull(volumeChoosingPolicy, "VolumeChoosingPolicy " +
"cannot be null");
Preconditions.checkNotNull(volumeSet, "VolumeSet cannot be null");
Expand All @@ -155,7 +156,7 @@ public void create(VolumeSet volumeSet, VolumeChoosingPolicy
while (true) {
HddsVolume containerVolume;
try {
containerVolume = volumeChoosingPolicy.chooseVolume(volumes, maxSize);
containerVolume = volumeChoosingPolicy.chooseVolume(volumes, maxSize, minFreeSpaceCalculator);
} catch (DiskOutOfSpaceException ex) {
throw new StorageContainerException("Container creation failed, " +
"due to disk out of space", ex, DISK_OUT_OF_SPACE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
Expand Down Expand Up @@ -150,6 +151,7 @@ public class KeyValueHandler extends Handler {
private final Striped<Lock> containerCreationLocks;
private static FaultInjector injector;
private final Clock clock;
private final MinFreeSpaceCalculator minFreeSpaceCalculator;

public KeyValueHandler(ConfigurationSource config,
String datanodeId,
Expand Down Expand Up @@ -214,6 +216,7 @@ public KeyValueHandler(ConfigurationSource config,
OzoneConfiguration.of(conf).set(ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY,
DEFAULT_LAYOUT.name());
}
this.minFreeSpaceCalculator = new MinFreeSpaceCalculator(conf);
}

@VisibleForTesting
Expand Down Expand Up @@ -427,7 +430,7 @@ ContainerCommandResponseProto handleCreateContainer(
containerIdLock.lock();
try {
if (containerSet.getContainer(containerID) == null) {
newContainer.create(volumeSet, volumeChoosingPolicy, clusterId);
newContainer.create(volumeSet, volumeChoosingPolicy, minFreeSpaceCalculator, clusterId);
if (RECOVERING == newContainer.getContainerState()) {
created = containerSet.addContainerByOverwriteMissingContainer(newContainer);
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.VolumeChoosingPolicyFactory;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
Expand All @@ -60,6 +61,7 @@ public class ContainerImporter {
private final ContainerController controller;
private final MutableVolumeSet volumeSet;
private final VolumeChoosingPolicy volumeChoosingPolicy;
private final MinFreeSpaceCalculator minFreeSpaceCalculator;
private final long containerSize;

private final Set<Long> importContainerProgress
Expand All @@ -83,6 +85,7 @@ public ContainerImporter(@Nonnull ConfigurationSource conf,
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
this.conf = conf;
this.minFreeSpaceCalculator = new MinFreeSpaceCalculator(conf);
}

public boolean isAllowedContainerImport(long containerID) {
Expand Down Expand Up @@ -149,7 +152,7 @@ HddsVolume chooseNextVolume() throws IOException {
// Choose volume that can hold both container in tmp and dest directory
return volumeChoosingPolicy.chooseVolume(
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList()),
containerSize * 2);
containerSize * 2, minFreeSpaceCalculator);
}

public static Path getUntarDirectory(HddsVolume hddsVolume)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
Expand Down Expand Up @@ -277,6 +278,8 @@ public static KeyValueContainer addContainerToVolumeDir(
throws IOException {
VolumeChoosingPolicy volumeChoosingPolicy =
new RoundRobinVolumeChoosingPolicy();
MinFreeSpaceCalculator minFreeSpaceCalculator =
new MinFreeSpaceCalculator(conf);
ContainerLayoutVersion layout = ContainerLayoutVersion.FILE_PER_BLOCK;

KeyValueContainerData keyValueContainerData = new KeyValueContainerData(
Expand All @@ -288,7 +291,7 @@ public static KeyValueContainer addContainerToVolumeDir(

KeyValueContainer container =
new KeyValueContainer(keyValueContainerData, conf);
container.create(volume.getVolumeSet(), volumeChoosingPolicy, clusterId);
container.create(volume.getVolumeSet(), volumeChoosingPolicy, minFreeSpaceCalculator, clusterId);

container.close();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
Expand Down Expand Up @@ -181,7 +182,7 @@ private KeyValueContainerData createToDeleteBlocks(ContainerSet containerSet,
data.setSchemaVersion(schemaVersion);
KeyValueContainer container = new KeyValueContainer(data, conf);
container.create(volumeSet,
new RoundRobinVolumeChoosingPolicy(), scmId);
new RoundRobinVolumeChoosingPolicy(), new MinFreeSpaceCalculator(conf), scmId);
containerSet.addContainer(container);
data = (KeyValueContainerData) containerSet.getContainer(
containerID).getContainerData();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
Expand Down Expand Up @@ -274,7 +275,7 @@ private KeyValueContainer createTestContainer() throws IOException {
cData.setSchemaVersion(OzoneConsts.SCHEMA_V2);
KeyValueContainer container = new KeyValueContainer(cData, conf);
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
clusterID);
new MinFreeSpaceCalculator(conf), clusterID);

// populate with some blocks
// metadata will be updated here, too
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.anyLong;
import static org.mockito.Mockito.mock;
Expand Down Expand Up @@ -53,6 +54,7 @@
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
Expand All @@ -79,6 +81,7 @@ public class TestStaleRecoveringContainerScrubbingService {
private int containerIdNum = 0;
private MutableVolumeSet volumeSet;
private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
private MinFreeSpaceCalculator freeSpaceCalculator;
private final TestClock testClock =
new TestClock(Instant.now(), ZoneOffset.UTC);

Expand All @@ -105,7 +108,8 @@ private void init() throws IOException {
volumeSet = mock(MutableVolumeSet.class);

volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class);
when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
freeSpaceCalculator = mock(MinFreeSpaceCalculator.class);
when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong(), any(MinFreeSpaceCalculator.class)))
.thenReturn(hddsVolume);
}

Expand Down Expand Up @@ -135,7 +139,7 @@ private List<Long> createTestContainers(
new KeyValueContainer(recoveringContainerData,
conf);
recoveringKeyValueContainer.create(
volumeSet, volumeChoosingPolicy, clusterID);
volumeSet, volumeChoosingPolicy, freeSpaceCalculator, clusterID);
containerSet.addContainer(recoveringKeyValueContainer);
createdIds.add((long) containerIdNum);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
Expand Down Expand Up @@ -117,6 +118,7 @@ public class TestContainerPersistence {
private static String hddsPath;
private static OzoneConfiguration conf;
private static VolumeChoosingPolicy volumeChoosingPolicy;
private static MinFreeSpaceCalculator freeSpaceCalculator;

private ContainerSet containerSet;
private MutableVolumeSet volumeSet;
Expand All @@ -143,6 +145,7 @@ public static void init() {
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, hddsPath);
conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, hddsPath);
volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
freeSpaceCalculator = new MinFreeSpaceCalculator(conf);
}

@AfterAll
Expand Down Expand Up @@ -204,7 +207,7 @@ private KeyValueContainer addContainer(ContainerSet cSet, long cID)
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
KeyValueContainer container = new KeyValueContainer(data, conf);
container.create(volumeSet, volumeChoosingPolicy, SCM_ID);
container.create(volumeSet, volumeChoosingPolicy, freeSpaceCalculator, SCM_ID);
commitBytesBefore = container.getContainerData()
.getVolume().getCommittedBytes();
cSet.addContainer(container);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage.MinFreeSpaceCalculator;
import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
Expand Down Expand Up @@ -140,7 +141,7 @@ public void testContainerCloseActionWhenFull(
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
.forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile()));
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
new MinFreeSpaceCalculator(conf), scmId.toString());
containerSet.addContainer(container);
ContainerMetrics metrics = ContainerMetrics.create(conf);
Map<ContainerType, Handler> handlers = Maps.newHashMap();
Expand Down Expand Up @@ -277,7 +278,7 @@ public void testContainerCloseActionWhenVolumeFull(
StorageVolumeUtil.getHddsVolumesList(volumeSet.getVolumesList())
.forEach(hddsVolume -> hddsVolume.setDbParentDir(tempDir.toFile()));
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(),
scmId.toString());
new MinFreeSpaceCalculator(conf), scmId.toString());
containerSet.addContainer(container);
ContainerMetrics metrics = ContainerMetrics.create(conf);
Map<ContainerType, Handler> handlers = Maps.newHashMap();
Expand Down Expand Up @@ -311,7 +312,8 @@ public void testContainerCloseActionWhenVolumeFull(
StorageContainerException scException =
assertThrows(StorageContainerException.class,
() -> container2.create(volumeSet,
new RoundRobinVolumeChoosingPolicy(), scmId.toString()));
new RoundRobinVolumeChoosingPolicy(), new MinFreeSpaceCalculator(conf),
scmId.toString()));
assertEquals("Container creation failed, due to disk out of space",
scException.getMessage());
} finally {
Expand Down
Loading