HDDS-13922. Increase available space requirement during pipeline/container allocation

.github/workflows/check.yml

@@ -247,7 +247,7 @@ jobs:
           if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then
             cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY
           fi
-          hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt
+          hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script }}/summary.txt || true
 
       - name: Archive build results
         if: ${{ !cancelled() }}

.github/workflows/intermittent-test-check.yml

@@ -223,7 +223,7 @@ jobs:
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Summary of failures
-        run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt || true
         if: ${{ !cancelled() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4

.github/workflows/repeat-acceptance.yml

@@ -150,7 +150,7 @@ jobs:
           KEEP_IMAGE: false
         continue-on-error: true
       - name: Summary of failures
-        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt || true
         if: ${{ !cancelled() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -359,6 +359,10 @@ public final class ScmConfigKeys {
       "ozone.scm.container.size";
   public static final String OZONE_SCM_CONTAINER_SIZE_DEFAULT = "5GB";
 
+  public static final String OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER =
+      "ozone.scm.container.space.requirement.multiplier";
+  public static final double OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT = 2.0;
+
   public static final String OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE =
       "ozone.scm.container.lock.stripes";
   public static final int OZONE_SCM_CONTAINER_LOCK_STRIPE_SIZE_DEFAULT = 512;

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1035,6 +1035,22 @@
       balances the amount of metadata.
     </description>
   </property>
+  <property>
+    <name>ozone.scm.container.space.requirement.multiplier</name>
+    <value>2.0</value>
+    <tag>OZONE, SCM, MANAGEMENT</tag>
+    <description>
+      Multiplier for container space requirement when checking if a datanode
+      has enough space for container allocation. The required space is calculated
+      as container size multiplied by this value. This prevents concurrent clients
+      from all passing the space check when there's only enough space for one
+      container. For example, with default container size of 5GB and multiplier
+      of 2.0, the system will require 10GB of available space before allocating
+      a new container. This ensures that if only 6GB is remaining, the check will
+      fail, preventing multiple clients from attempting to create containers
+      concurrently when there's only space for one.
+    </description>
+  </property>
   <property>
     <name>ozone.scm.container.lock.stripes</name>
     <value>512</value>

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java

@@ -82,6 +82,7 @@ public class ContainerManagerImpl implements ContainerManager {
   private final Random random = new Random();
 
   private final long maxContainerSize;
+  private final long containerSpaceRequirement;
 
   /**
    *
@@ -115,6 +116,11 @@ public ContainerManagerImpl(
     maxContainerSize = (long) conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
         ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
 
+    double multiplier = conf.getDouble(
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER,
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT);
+    this.containerSpaceRequirement = (long) (maxContainerSize * multiplier);
+
     this.scmContainerManagerMetrics = SCMContainerManagerMetrics.create();
   }
 
@@ -352,23 +358,23 @@ public ContainerInfo getMatchingContainer(final long size, final String owner,
       synchronized (pipeline.getId()) {
         containerIDs = getContainersForOwner(pipeline, owner);
         if (containerIDs.size() < getOpenContainerCountPerPipeline(pipeline)) {
-          if (pipelineManager.hasEnoughSpace(pipeline, maxContainerSize)) {
+          if (pipelineManager.hasEnoughSpace(pipeline, containerSpaceRequirement)) {
             allocateContainer(pipeline, owner);
             containerIDs = getContainersForOwner(pipeline, owner);
           } else {
             LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.",
-                pipeline, maxContainerSize);
+                pipeline, containerSpaceRequirement);
           }
         }
         containerIDs.removeAll(excludedContainerIDs);
         containerInfo = containerStateManager.getMatchingContainer(
             size, owner, pipeline.getId(), containerIDs);
         if (containerInfo == null) {
-          if (pipelineManager.hasEnoughSpace(pipeline, maxContainerSize)) {
+          if (pipelineManager.hasEnoughSpace(pipeline, containerSpaceRequirement)) {
             containerInfo = allocateContainer(pipeline, owner);
           } else {
             LOG.debug("Cannot allocate a new container because pipeline {} does not have the required space {}.",
-                pipeline, maxContainerSize);
+                pipeline, containerSpaceRequirement);
           }
         }
         return containerInfo;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java

@@ -62,6 +62,7 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> {
   private final ConfigurationSource conf;
   private final PlacementPolicy placementPolicy;
   private final long containerSizeBytes;
+  private final long containerSpaceRequirement;
 
   public ECPipelineProvider(NodeManager nodeManager,
                             PipelineStateManager stateManager,
@@ -73,6 +74,10 @@ public ECPipelineProvider(NodeManager nodeManager,
     this.containerSizeBytes = (long) this.conf
         .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
             ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+    double multiplier = conf.getDouble(
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER,
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT);
+    this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier);
   }
 
   @Override
@@ -88,7 +93,7 @@ protected Pipeline create(ECReplicationConfig replicationConfig,
       throws IOException {
     List<DatanodeDetails> dns = placementPolicy
         .chooseDatanodes(excludedNodes, favoredNodes,
-            replicationConfig.getRequiredNodes(), 0, this.containerSizeBytes);
+            replicationConfig.getRequiredNodes(), 0, this.containerSpaceRequirement);
     return create(replicationConfig, dns);
   }
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java

@@ -65,6 +65,7 @@ public class RatisPipelineProvider
   private final LeaderChoosePolicy leaderChoosePolicy;
   private final SCMContext scmContext;
   private final long containerSizeBytes;
+  private final long containerSpaceRequirement;
   private final long minRatisVolumeSizeBytes;
 
   @VisibleForTesting
@@ -89,6 +90,10 @@ public RatisPipelineProvider(NodeManager nodeManager,
         ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
         ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
         StorageUnit.BYTES);
+    double multiplier = conf.getDouble(
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER,
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT);
+    this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier);
     this.minRatisVolumeSizeBytes = (long) this.conf.getStorageSize(
         ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
         ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT,
@@ -165,7 +170,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig,
     switch (factor) {
     case ONE:
       dns = pickNodesNotUsed(replicationConfig, minRatisVolumeSizeBytes,
-          containerSizeBytes, conf);
+          containerSpaceRequirement, conf);
       break;
     case THREE:
       List<DatanodeDetails> excludeDueToEngagement = filterPipelineEngagement();
@@ -178,7 +183,7 @@ public synchronized Pipeline create(RatisReplicationConfig replicationConfig,
       }
       dns = placementPolicy.chooseDatanodes(excludedNodes,
           favoredNodes, factor.getNumber(), minRatisVolumeSizeBytes,
-          containerSizeBytes);
+          containerSpaceRequirement);
       break;
     default:
       throw new IllegalStateException("Unknown factor: " + factor.name());

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java

@@ -27,8 +27,11 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.atLeast;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -47,6 +50,7 @@
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps;
 import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap;
 import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
@@ -179,6 +183,42 @@ public void testGetMatchingContainerReturnsContainerWhenEnoughSpaceInDatanodes()
     assertNotNull(container);
   }
 
+  @Test
+  public void testContainerSpaceRequirementMultiplier() throws IOException {
+    long sizeRequired = 256 * 1024 * 1024;
+    double multiplier = 2.0;
+    long containerSize = 5L * 1024 * 1024 * 1024;
+    long expectedSpaceRequirement = (long) (containerSize * multiplier);
+
+    PipelineManager spyPipelineManager = spy(pipelineManager);
+    File tempDir = new File(testDir, "tempDir");
+    OzoneConfiguration conf = SCMTestUtils.getConf(tempDir);
+    conf.setDouble(ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, multiplier);
+
+    ContainerManager manager = new ContainerManagerImpl(conf,
+        scmhaManager, sequenceIdGen, spyPipelineManager,
+        SCMDBDefinition.CONTAINERS.getTable(dbStore), pendingOpsMock);
+
+    Pipeline pipeline = spyPipelineManager.getPipelines().iterator().next();
+
+    doReturn(false).when(spyPipelineManager)
+        .hasEnoughSpace(any(Pipeline.class), anyLong());
+    ContainerInfo container = manager
+        .getMatchingContainer(sizeRequired, "test", pipeline, Collections.emptySet());
+    assertNull(container);
+    verify(spyPipelineManager, atLeast(1))
+        .hasEnoughSpace(eq(pipeline), eq(expectedSpaceRequirement));
+
+    reset(spyPipelineManager);
+    doReturn(true).when(spyPipelineManager)
+        .hasEnoughSpace(any(Pipeline.class), anyLong());
+    container = manager
+        .getMatchingContainer(sizeRequired, "test", pipeline, Collections.emptySet());
+    assertNotNull(container);
+    verify(spyPipelineManager, atLeast(1))
+        .hasEnoughSpace(eq(pipeline), eq(expectedSpaceRequirement));
+  }
+
   @Test
   void testUpdateContainerState() throws Exception {
     final ContainerInfo container = containerManager.allocateContainer(

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java

@@ -68,17 +68,21 @@ public class TestECPipelineProvider {
   private PipelineStateManager stateManager =
       mock(PipelineStateManager.class);
   private PlacementPolicy placementPolicy = mock(PlacementPolicy.class);
-  private long containerSizeBytes;
+  private long containerSpaceRequirement;
 
   @BeforeEach
   public void setup() throws IOException, NodeNotFoundException {
     OzoneConfiguration conf = new OzoneConfiguration();
     provider = new ECPipelineProvider(
         nodeManager, stateManager, conf, placementPolicy);
-    this.containerSizeBytes = (long) conf.getStorageSize(
+    long containerSizeBytes = (long) conf.getStorageSize(
         ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
         ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
         StorageUnit.BYTES);
+    double multiplier = conf.getDouble(
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER,
+        ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER_DEFAULT);
+    this.containerSpaceRequirement = (long) (containerSizeBytes * multiplier);
     // Placement policy will always return EC number of random nodes.
     when(placementPolicy.chooseDatanodes(anyList(),
         anyList(), anyInt(), anyLong(),
@@ -200,7 +204,7 @@ public void testExcludedAndFavoredNodesPassedToPlacementPolicy()
     assertEquals(ecConf.getData() + ecConf.getParity(), pipeline.getNodes().size());
 
     verify(placementPolicy).chooseDatanodes(excludedNodes, favoredNodes,
-        ecConf.getRequiredNodes(), 0, containerSizeBytes);
+        ecConf.getRequiredNodes(), 0, containerSpaceRequirement);
   }
 
   private Set<ContainerReplica> createContainerReplicas(int number) {

hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml

@@ -31,6 +31,7 @@ data:
   OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB"
+  OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0"
   OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata"
   OZONE-SITE.XML_ozone.om.address: "om-0.om"
   OZONE-SITE.XML_ozone.recon.address: "recon-0.recon"

hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml

@@ -31,6 +31,7 @@ data:
   OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB"
+  OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.recon.address: recon-0.recon

hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml

@@ -31,6 +31,7 @@ data:
   OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB"
+  OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.recon.address: recon-0.recon

hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml

@@ -31,6 +31,7 @@ data:
   OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB"
+  OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.recon.address: recon-0.recon

hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml

@@ -31,6 +31,7 @@ data:
   OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB"
+  OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.recon.address: recon-0.recon

hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml

@@ -31,6 +31,7 @@ data:
   OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
   OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
   OZONE-SITE.XML_hdds.datanode.volume.min.free.space: "1GB"
+  OZONE-SITE.XML_ozone.scm.container.space.requirement.multiplier: "1.0"
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.om.address: om-0.om
   OZONE-SITE.XML_ozone.recon.address: recon-0.recon

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java

@@ -99,6 +99,7 @@ public void init() throws Exception {
     conf = new OzoneConfiguration();
     chunkSize = (int) OzoneConsts.MB;
     blockSize = 4 * chunkSize;
+    conf.setDouble(ScmConfigKeys.OZONE_SCM_CONTAINER_SPACE_REQUIREMENT_MULTIPLIER, 1.0);
     conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 100, TimeUnit.SECONDS);
 
     RatisClientConfig ratisClientConfig =