HDDS-6901. Configure HDDS volume reserved as percentage of the volume space.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -214,6 +214,8 @@ public final class ScmConfigKeys {
   public static final String HDDS_DATANODE_DIR_KEY = "hdds.datanode.dir";
   public static final String HDDS_DATANODE_DIR_DU_RESERVED =
       "hdds.datanode.dir.du.reserved";
+  public static final String HDDS_DATANODE_VOLUME_RESERVED =
+      "hdds.datanode.volume.reserved";
   public static final String HDDS_REST_CSRF_ENABLED_KEY =
       "hdds.rest.rest-csrf.enabled";
   public static final boolean HDDS_REST_CSRF_ENABLED_DEFAULT = false;

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -166,6 +166,14 @@
        Such as /dir1:100B, /dir2:200MB, means dir1 reserves 100 bytes and dir2 reserves 200 MB.
     </description>
   </property>
+  <property>
+    <name>hdds.datanode.volume.reserved</name>
+    <value/>
+    <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT</tag>
+    <description>Percentage of volume that should be reserved. This space is left free for other usage.
+      The value should be between 0-1. Such as 0.1 which means 10% of volume space will be reserved.
+    </description>
+  </property>
   <property>
     <name>hdds.datanode.volume.choosing.policy</name>
     <value/>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java

@@ -33,6 +33,7 @@
 import org.slf4j.LoggerFactory;
 
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_VOLUME_RESERVED;
 
 /**
  * Stores information about a disk/volume.
@@ -134,25 +135,45 @@ public VolumeInfo build() throws IOException {
   }
 
   private long getReserved(ConfigurationSource conf) {
+    final float defaultValue = Float.MAX_VALUE;
+    float percentage = conf.getFloat(HDDS_DATANODE_VOLUME_RESERVED,
+        defaultValue);
     Collection<String> reserveList = conf.getTrimmedStringCollection(
         HDDS_DATANODE_DIR_DU_RESERVED);
-    for (String reserve : reserveList) {
-      String[] words = reserve.split(":");
-      if (words.length < 2) {
-        LOG.error("Reserved space should config in pair, but current is {}",
-            reserve);
-        continue;
-      }
+    //Both the configs are set. Log it and return 0
+    if (reserveList.size() > 0 && percentage != defaultValue) {
+      LOG.error("Both {} and {} are set. Set either one, not both. " +
+          "Setting reserved to 0.", HDDS_DATANODE_VOLUME_RESERVED,
+          HDDS_DATANODE_DIR_DU_RESERVED);
+      return 0;
+    }
 
-      if (words[0].trim().equals(rootDir)) {
-        try {
-          StorageSize size = StorageSize.parse(words[1].trim());
-          return (long) size.getUnit().toBytes(size.getValue());
-        } catch (Exception e) {
-          LOG.error("Failed to parse StorageSize:{}", words[1].trim(), e);
-          return 0;
+    if (reserveList.size() > 0) {
+      for (String reserve : reserveList) {
+        String[] words = reserve.split(":");
+        if (words.length < 2) {
+          LOG.error("Reserved space should config in pair, but current is {}",
+              reserve);
+          continue;
         }
+
+        if (words[0].trim().equals(rootDir)) {
+          try {
+            StorageSize size = StorageSize.parse(words[1].trim());
+            return (long) size.getUnit().toBytes(size.getValue());
+          } catch (Exception e) {
+            LOG.error("Failed to parse StorageSize:{}", words[1].trim(), e);
+            return 0;
+          }
+        }
+      }
+    } else if (percentage != defaultValue) {
+      if (0 <= percentage && percentage <= 1) {
+        return (long) Math.ceil(this.usage.getCapacity() * percentage);
       }
+      //If it comes here then the percentage is not between 0-1.
+      LOG.error("The value of {} should be between 0 to 1. Defaulting to 0.",
+          HDDS_DATANODE_VOLUME_RESERVED);
     }
 
     return 0;
@@ -183,8 +204,9 @@ private VolumeInfo(Builder b) throws IOException {
     SpaceUsageCheckParams checkParams =
         usageCheckFactory.paramsFor(root);
 
+    this.usage = new VolumeUsage(checkParams);
     this.reservedInBytes = getReserved(b.conf);
-    this.usage = new VolumeUsage(checkParams, reservedInBytes);
+    this.usage.setReserved(reservedInBytes);
   }
 
   public long getCapacity() {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java

@@ -30,10 +30,9 @@ public class VolumeUsage implements SpaceUsageSource {
 
   private final CachingSpaceUsageSource source;
   private boolean shutdownComplete;
-  private final long reservedInBytes;
+  private long reservedInBytes;
 
-  VolumeUsage(SpaceUsageCheckParams checkParams, long reservedInBytes) {
-    this.reservedInBytes = reservedInBytes;
+  VolumeUsage(SpaceUsageCheckParams checkParams) {
     source = new CachingSpaceUsageSource(checkParams);
     start(); // TODO should start only on demand
   }
@@ -98,4 +97,8 @@ public synchronized void shutdown() {
   public void refreshNow() {
     source.refreshNow();
   }
+
+  public void setReserved(long reserved) {
+    this.reservedInBytes = reserved;
+  }
 }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java

@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.volume;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.UUID;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_VOLUME_RESERVED;
+
+/**
+ * To test the reserved volume space.
+ */
+public class TestReservedVolumeSpace {
+
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+  private static final String DATANODE_UUID = UUID.randomUUID().toString();
+  private HddsVolume.Builder volumeBuilder;
+
+  @Before
+  public void setup() throws Exception {
+    volumeBuilder = new HddsVolume.Builder(folder.getRoot().getPath())
+        .datanodeUuid(DATANODE_UUID)
+        .usageCheckFactory(MockSpaceUsageCheckFactory.NONE);
+  }
+
+  /**
+   * Test reserved capacity with respect to the percentage of actual capacity.
+   * @throws Exception
+   */
+  @Test
+  public void testVolumeCapacityAfterReserve() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(HDDS_DATANODE_VOLUME_RESERVED, "0.3");
+    HddsVolume hddsVolume = volumeBuilder.conf(conf).build();
+    //Reserving
+    float percentage = conf.getFloat(HDDS_DATANODE_VOLUME_RESERVED, -1);
+
+    long volumeCapacity = hddsVolume.getCapacity();
+    //Gets the actual total capacity
+    long totalCapacity = hddsVolume.getVolumeInfo()
+        .getUsageForTesting().getCapacity();
+    long reservedCapacity = hddsVolume.getVolumeInfo().getReservedInBytes();
+    //Volume Capacity with Reserved
+    long volumeCapacityReserved = totalCapacity - reservedCapacity;
+
+    long reservedFromVolume = hddsVolume.getVolumeInfo().getReservedInBytes();
+    long reservedCalculated = (long) Math.ceil(totalCapacity * percentage);
+
+    Assert.assertEquals(reservedFromVolume, reservedCalculated);
+    Assert.assertEquals(volumeCapacity, volumeCapacityReserved);
+  }
+
+  /**
+   * Either hdds.datanode.volume.reserved or hdds.datanode.dir.du.reserved
+   * should be set. But not both.
+   * @throws Exception
+   */
+  @Test
+  public void testReservedToZeroWhenBothConfigSet() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(HDDS_DATANODE_VOLUME_RESERVED, "0.3");
+    conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED,
+        folder.getRoot() + ":500B");
+    HddsVolume hddsVolume = volumeBuilder.conf(conf).build();
+
+    long reservedFromVolume = hddsVolume.getVolumeInfo().getReservedInBytes();
+    //When both the configs hdds.datanode.volume.reserved and
+    //hdds.datanode.dir.du.reserved are set. We set reserved to 0
+    Assert.assertEquals(reservedFromVolume, 0);
+  }
+
+  @Test
+  public void testReservedToZeroWhenBothConfigNotSet() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    HddsVolume hddsVolume = volumeBuilder.conf(conf).build();
+
+    long reservedFromVolume = hddsVolume.getVolumeInfo().getReservedInBytes();
+    Assert.assertEquals(reservedFromVolume, 0);
+  }
+}