HDDS-11513. All deletion configurations should be configurable without restart

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -22,6 +22,8 @@
 import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser;
 import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS;
 import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
 import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
@@ -291,6 +293,10 @@ public String getNamespace() {
                   this::reconfigBlockDeleteThreadMax)
               .register(OZONE_BLOCK_DELETING_SERVICE_WORKERS,
                   this::reconfigDeletingServiceWorkers)
+              .register(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
+                  this::reconfigBlockDeletingServiceInterval)
+              .register(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
+                  this::reconfigBlockDeletingServiceTimeout)
               .register(REPLICATION_STREAMS_LIMIT_KEY,
                   this::reconfigReplicationStreamsLimit);
 
@@ -676,6 +682,22 @@ private String reconfigReplicationStreamsLimit(String value) {
     return value;
   }
 
+  private String reconfigBlockDeletingServiceInterval(String value) {
+    getConf().set(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, value);
+
+    getDatanodeStateMachine().getContainer().getBlockDeletingService()
+        .setBlockDeletingServiceInterval(value);
+    return value;
+  }
+
+  private String reconfigBlockDeletingServiceTimeout(String value) {
+    getConf().set(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT, value);
+
+    getDatanodeStateMachine().getContainer().getBlockDeletingService()
+        .setBlockDeletingServiceTimeout(value);
+    return value;
+  }
+
   /**
    * Returns the initial version of the datanode.
    */

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/BlockDeletingService.java

@@ -35,6 +35,7 @@
 import org.apache.hadoop.hdds.utils.BackgroundService;
 import org.apache.hadoop.hdds.utils.BackgroundTask;
 import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.container.common.helpers.BlockDeletingServiceMetrics;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
@@ -64,6 +65,8 @@ public class BlockDeletingService extends BackgroundService {
   private static final int TASK_PRIORITY_DEFAULT = 1;
 
   private final Duration blockDeletingMaxLockHoldingTime;
+  private String blockDeletingServiceInterval;
+  private String blockDeletingServiceTimeout;
 
   @VisibleForTesting
   public BlockDeletingService(
@@ -99,6 +102,12 @@ public BlockDeletingService(
     this.blockDeletingMaxLockHoldingTime =
         dnConf.getBlockDeletingMaxLockHoldingTime();
     metrics = BlockDeletingServiceMetrics.create();
+    this.blockDeletingServiceInterval = conf.get(
+        OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
+        OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT);
+    this.blockDeletingServiceTimeout = conf.get(
+        OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
+        OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT);
   }
 
   /**
@@ -272,6 +281,20 @@ public int getBlockLimitPerInterval() {
     return dnConf.getBlockDeletionLimit();
   }
 
+  public String getBlockDeletingServiceInterval() {
+    return blockDeletingServiceInterval;
+  }
+  public void setBlockDeletingServiceInterval(String blockDeletingServiceInterval) {
+    this.blockDeletingServiceInterval = blockDeletingServiceInterval;
+  }
+
+  public String getBlockDeletingServiceTimeout() {
+    return blockDeletingServiceTimeout;
+  }
+  public void setBlockDeletingServiceTimeout(String blockDeletingServiceTimeout) {
+    this.blockDeletingServiceTimeout = blockDeletingServiceTimeout;
+  }
+
   private static class BlockDeletingTaskBuilder {
     private BlockDeletingService blockDeletingService;
     private BlockDeletingService.ContainerBlockInfo containerBlockInfo;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestDatanodeReconfiguration.java

@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.ozone.reconfig;
 
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS;
 import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX;
 import static org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig.REPLICATION_STREAMS_LIMIT_KEY;
@@ -49,6 +51,8 @@ void reconfigurableProperties() {
     Set<String> expected = ImmutableSet.<String>builder()
         .add(HDDS_DATANODE_BLOCK_DELETE_THREAD_MAX)
         .add(OZONE_BLOCK_DELETING_SERVICE_WORKERS)
+        .add(OZONE_BLOCK_DELETING_SERVICE_INTERVAL)
+        .add(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT)
         .add(REPLICATION_STREAMS_LIMIT_KEY)
         .addAll(new DatanodeConfiguration().reconfigurableProperties())
         .build();
@@ -92,6 +96,26 @@ void blockDeletingServiceWorkers(int delta) throws ReconfigurationException {
     assertEquals(newValue, executor.getCorePoolSize());
   }
 
+  @Test
+  void blockDeletingServiceInterval() throws ReconfigurationException {
+    //Initial string is 1m
+    getFirstDatanode().getReconfigurationHandler().reconfigurePropertyImpl(
+        OZONE_BLOCK_DELETING_SERVICE_INTERVAL, "2m");
+
+    assertEquals("2m", getFirstDatanode().getDatanodeStateMachine().getContainer()
+        .getBlockDeletingService().getBlockDeletingServiceInterval());
+  }
+
+  @Test
+  void blockDeletingServiceTimeout() throws ReconfigurationException {
+    //Initial string is 300000ms
+    getFirstDatanode().getReconfigurationHandler().reconfigurePropertyImpl(
+        OZONE_BLOCK_DELETING_SERVICE_TIMEOUT, "350000ms");
+
+    assertEquals("350000ms", getFirstDatanode().getDatanodeStateMachine().getContainer()
+        .getBlockDeletingService().getBlockDeletingServiceTimeout());
+  }
+
   @ParameterizedTest
   @ValueSource(ints = { -1, +1 })
   void replicationStreamsLimit(int delta) throws ReconfigurationException {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java

@@ -20,9 +20,11 @@
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_READONLY_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import com.google.common.collect.ImmutableSet;
@@ -51,6 +53,8 @@ void reconfigurableProperties() {
         .add(OZONE_KEY_DELETING_LIMIT_PER_TASK)
         .add(OZONE_OM_VOLUME_LISTALL_ALLOWED)
         .add(OZONE_READONLY_ADMINISTRATORS)
+        .add(OZONE_DIR_DELETING_SERVICE_INTERVAL)
+        .add(OZONE_THREAD_NUMBER_DIR_DELETION)
         .addAll(new OmConfig().reconfigurableProperties())
         .build();
 
@@ -121,4 +125,23 @@ void unsetAllowListAllVolumes(String newValue) throws ReconfigurationException {
     assertEquals(OZONE_OM_VOLUME_LISTALL_ALLOWED_DEFAULT, cluster().getOzoneManager().getAllowListAllVolumes());
   }
 
+  @Test
+  void dirDeletingServiceInterval() throws ReconfigurationException {
+    //Initial string is 1m
+    getSubject().reconfigurePropertyImpl(OZONE_DIR_DELETING_SERVICE_INTERVAL, "2m");
+
+    assertEquals("2m", cluster().getOzoneManager().
+        getKeyManager().getDirDeletingService().getDirDeletingServiceInterval());
+  }
+
+  @Test
+  void threadNumberDirDeletion() throws ReconfigurationException {
+    int initialNum = cluster().getOzoneManager().getKeyManager().getDirDeletingService().getThreadNumberDirDeletion();
+
+    getSubject().reconfigurePropertyImpl(OZONE_THREAD_NUMBER_DIR_DELETION, String.valueOf(initialNum + 10));
+
+    assertEquals(initialNum + 10, cluster().getOzoneManager().
+        getKeyManager().getDirDeletingService().getThreadNumberDirDeletion());
+  }
+
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -52,6 +52,7 @@
 import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
@@ -77,6 +78,7 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DETECTED_LOOP_IN_BUCKET_LINKS;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FEATURE_NOT_ENABLED;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR;
@@ -531,7 +533,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
                 this::reconfOzoneReadOnlyAdmins)
             .register(OZONE_OM_VOLUME_LISTALL_ALLOWED, this::reconfigureAllowListAllVolumes)
             .register(OZONE_KEY_DELETING_LIMIT_PER_TASK,
-                this::reconfOzoneKeyDeletingLimitPerTask);
+                this::reconfOzoneKeyDeletingLimitPerTask)
+            .register(OZONE_DIR_DELETING_SERVICE_INTERVAL, this::reconfOzoneDirDeletingServiceInterval)
+            .register(OZONE_THREAD_NUMBER_DIR_DELETION, this::reconfOzoneThreadNumberDirDeletion);
 
     versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion());
     upgradeFinalizer = new OMUpgradeFinalizer(versionManager);
@@ -4995,6 +4999,20 @@ private String reconfigureAllowListAllVolumes(String newVal) {
     return String.valueOf(allowListAllVolumes);
   }
 
+  private String reconfOzoneDirDeletingServiceInterval(String newVal) {
+    getConfiguration().set(OZONE_DIR_DELETING_SERVICE_INTERVAL, newVal);
+    getKeyManager().getDirDeletingService().setDirDeletingServiceInterval(newVal);
+    return newVal;
+  }
+
+  private String reconfOzoneThreadNumberDirDeletion(String newVal) {
+    Preconditions.checkArgument(Integer.parseInt(newVal) >= 0,
+        OZONE_THREAD_NUMBER_DIR_DELETION + " cannot be negative.");
+    getConfiguration().set(OZONE_THREAD_NUMBER_DIR_DELETION, newVal);
+    getKeyManager().getDirDeletingService().setThreadNumberDirDeletion(Integer.parseInt(newVal));
+    return newVal;
+  }
+
   public void validateReplicationConfig(ReplicationConfig replicationConfig)
       throws OMException {
     try {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java

@@ -80,6 +80,8 @@ public class DirectoryDeletingService extends AbstractKeyDeletingService {
   private int ratisByteLimit;
   private final AtomicBoolean suspended;
   private AtomicBoolean isRunningOnAOS;
+  private String dirDeletingServiceInterval;
+  private int threadNumberDirDeletion;
 
   private final DeletedDirSupplier deletedDirSupplier;
 
@@ -99,6 +101,12 @@ public DirectoryDeletingService(long interval, TimeUnit unit,
     this.suspended = new AtomicBoolean(false);
     this.isRunningOnAOS = new AtomicBoolean(false);
     this.dirDeletingCorePoolSize = dirDeletingServiceCorePoolSize;
+    this.dirDeletingServiceInterval = configuration.get(
+        OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL,
+        OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT);
+    this.threadNumberDirDeletion = configuration.getInt(
+        OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION,
+        OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION_DEFAULT);
     deletedDirSupplier = new DeletedDirSupplier();
     taskCount.set(0);
   }
@@ -346,4 +354,20 @@ public KeyValue<String, OmKeyInfo> getPendingDeletedDirInfo()
     return deletedDirSupplier.get();
   }
 
+  public String getDirDeletingServiceInterval() {
+    return dirDeletingServiceInterval;
+  }
+
+  public void setDirDeletingServiceInterval(String dirDeletingServiceInterval) {
+    this.dirDeletingServiceInterval = dirDeletingServiceInterval;
+  }
+
+  public int getThreadNumberDirDeletion() {
+    return threadNumberDirDeletion;
+  }
+
+  public void setThreadNumberDirDeletion(int threadNumberDirDeletion) {
+    this.threadNumberDirDeletion = threadNumberDirDeletion;
+  }
+
 }