From 121a084cb634a2996922bf5ba0bbce86a16f6b9f Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 9 Jul 2025 13:08:02 +0530 Subject: [PATCH] HDDS-11514. Set optimal default values for delete configurations based on live cluster testing. --- .../src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- .../common/statemachine/DatanodeConfiguration.java | 6 +++--- .../main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java index dbab95efd4bd..bc644247a2df 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java @@ -104,7 +104,7 @@ public class ScmConfig extends ReconfigurableConfig { @Config(key = "hdds.scm.block.deletion.per-interval.max", type = ConfigType.INT, - defaultValue = "100000", + defaultValue = "500000", reconfigurable = true, tags = { ConfigTag.SCM, ConfigTag.DELETION}, description = diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f240d0cfbd45..0418673bd912 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -467,7 +467,7 @@ ozone.key.deleting.limit.per.task - 20000 + 50000 OM, PERFORMANCE A maximum number of keys to be scanned by key deleting service diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 7d1c0543d0bd..6d2b92831476 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -258,13 +258,13 @@ public class DatanodeConfiguration extends ReconfigurableConfig { private long blockDeleteMaxLockWaitTimeoutMs = Duration.ofMillis(100).toMillis(); @Config(key = "block.deleting.limit.per.interval", - defaultValue = "5000", + defaultValue = "20000", reconfigurable = true, type = ConfigType.INT, - tags = { ConfigTag.SCM, ConfigTag.DELETION }, + tags = { ConfigTag.SCM, ConfigTag.DELETION, DATANODE }, description = "Number of blocks to be deleted in an interval." ) - private int blockLimitPerInterval = 5000; + private int blockLimitPerInterval = 20000; @Config(key = "block.deleting.max.lock.holding.time", defaultValue = "1s", diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 5314a8488455..751e0a669f66 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -106,7 +106,7 @@ public final class OMConfigKeys { public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK = "ozone.key.deleting.limit.per.task"; - public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 20000; + public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 50000; public static final String OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK = "ozone.snapshot.key.deleting.limit.per.task"; public static final int OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT