diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 627c432d3c5d..dd668caaadb6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -172,23 +172,6 @@ public final class OzoneConfigKeys { public static final String OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT = "60s"; - /** - * The interval of open key clean service. - */ - public static final String OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS = - "ozone.open.key.cleanup.service.interval.seconds"; - public static final int - OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT - = 24 * 3600; // a total of 24 hour - - /** - * An open key gets cleaned up when it is being in open state for too long. - */ - public static final String OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS = - "ozone.open.key.expire.threshold"; - public static final int OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT = - 24 * 3600; - public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT = "ozone.block.deleting.service.timeout"; public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 24f0c454c2ff..53f490e7964d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1188,23 +1188,36 @@ - ozone.open.key.cleanup.service.interval.seconds - 86400 + ozone.om.open.key.cleanup.service.interval + 24h OZONE, OM, PERFORMANCE - A background job periodically checks open key entries and delete the expired ones. This entry controls the - interval of this cleanup check. + A background job that periodically checks open key entries and marks + expired open keys for deletion. This entry controls the interval of this + cleanup check. Unit could be defined with postfix (ns,ms,s,m,h,d) - ozone.open.key.expire.threshold - 86400 + ozone.om.open.key.expire.threshold + 7d OZONE, OM, PERFORMANCE Controls how long an open key operation is considered active. Specifically, if a key has been open longer than the value of this config entry, that open key is considered as - expired (e.g. due to client crash). Default to 24 hours. + expired (e.g. due to client crash). Unit could be defined with postfix (ns,ms,s,m,h,d) + + + + + ozone.om.open.key.cleanup.limit.per.task + 1000 + OZONE, OM, PERFORMANCE + + The maximum number of open keys to be identified as expired and marked + for deletion by one run of the open key cleanup service on the OM. + This property is used to throttle the actual number of open key deletions + on the OM. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 82d26f95ac4a..eacb3f7959f5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -83,6 +83,21 @@ private OMConfigKeys() { "ozone.key.deleting.limit.per.task"; public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 20000; + public static final String OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL = + "ozone.om.open.key.cleanup.service.interval"; + public static final String + OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_DEFAULT = "24h"; + + public static final String OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD = + "ozone.om.open.key.expire.threshold"; + public static final String OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD_DEFAULT = + "7d"; + + public static final String OZONE_OM_OPEN_KEY_CLEANUP_LIMIT_PER_TASK = + "ozone.om.open.key.cleanup.limit.per.task"; + public static final int OZONE_OM_OPEN_KEY_CLEANUP_LIMIT_PER_TASK_DEFAULT = + 1000; + public static final String OZONE_OM_METRICS_SAVE_INTERVAL = "ozone.om.save.metrics.interval"; public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m"; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java index dfd558ff2246..ca8e4799eec1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.Collections; import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics; @@ -38,8 +39,9 @@ import org.apache.commons.io.FileUtils; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; + import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -84,7 +86,8 @@ public void init() throws Exception { scmId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString(); conf.setBoolean(OZONE_ACL_ENABLED, true); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); + conf.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, + 2, TimeUnit.SECONDS); cluster = MiniOzoneCluster.newBuilder(conf) .setClusterId(clusterId) .setScmId(scmId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java index fe8798466e16..2aaf78e13a03 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java @@ -31,12 +31,13 @@ import java.io.IOException; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; /** * This class tests MiniOzoneHAClusterImpl. @@ -70,7 +71,8 @@ public void init() throws Exception { conf.setBoolean(OZONE_ACL_ENABLED, true); conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); + conf.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, + 2, TimeUnit.SECONDS); cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(clusterId) .setScmId(scmId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index d735a0ced650..ff7b3213e3a2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -47,7 +47,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.writeDBCheckpointToStream; @@ -167,7 +166,6 @@ private void setupCluster() throws Exception { public void testDoGet() throws Exception { conf.setBoolean(OZONE_ACL_ENABLED, false); conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); setupCluster(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java index 3cac66a91c50..fd43a0fa26d5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java @@ -39,7 +39,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import org.junit.AfterClass; import static org.junit.Assert.assertTrue; @@ -88,7 +87,6 @@ public static void init() throws Exception { scmId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString(); conf.setBoolean(OZONE_ACL_ENABLED, true); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); conf.setClass(OZONE_ACL_AUTHORIZER_CLASS, OzoneAccessAuthorizerTest.class, IAccessAuthorizer.class); conf.setStrings(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java index ea3240803d7b..aaf07e8b7d23 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmInit.java @@ -18,12 +18,14 @@ import java.io.IOException; import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; + import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -65,7 +67,8 @@ public static void init() throws Exception { clusterId = UUID.randomUUID().toString(); scmId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString(); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); + conf.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, + 2, TimeUnit.SECONDS); cluster = MiniOzoneCluster.newBuilder(conf) .setClusterId(clusterId) .setScmId(scmId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java index 7a10ef52beec..49dea0f0061a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java @@ -55,7 +55,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; @@ -141,7 +140,6 @@ public static void init() throws Exception { conf.setBoolean(OZONE_ACL_ENABLED, true); conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); conf.setInt(OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java index 33d3f12e61f3..1bc892fd83fc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumes.java @@ -43,7 +43,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_VOLUME_LISTALL_ALLOWED; import static org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType.OZONE; @@ -92,7 +91,6 @@ public static void setupClass() String clusterId = UUID.randomUUID().toString(); String scmId = UUID.randomUUID().toString(); String omId = UUID.randomUUID().toString(); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10); // Use native impl here, default impl doesn't do actual checks diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java index c23595fce62a..ddc06f6a04ba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestart.java @@ -43,7 +43,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import org.junit.AfterClass; import org.junit.Assert; @@ -83,7 +82,6 @@ public static void init() throws Exception { scmId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString(); conf.setBoolean(OZONE_ACL_ENABLED, true); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD); conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java index 24d27395cc4b..c94970cad413 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java @@ -43,14 +43,15 @@ import java.security.PublicKey; import java.security.cert.X509Certificate; import java.util.UUID; +import java.util.concurrent.TimeUnit; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.apache.ozone.test.GenericTestUtils.LogCapturer; import static org.apache.ozone.test.GenericTestUtils.getTempPath; @@ -84,7 +85,8 @@ public void init() throws Exception { omId = UUID.randomUUID().toString(); conf.setBoolean(OZONE_ACL_ENABLED, true); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); + conf.setTimeDuration(OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, + 2, TimeUnit.SECONDS); conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.toString()); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 2); conf.set(OZONE_SCM_NAMES, "localhost"); diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index f565a5d49a26..aee7d78957b0 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om; import java.io.IOException; +import java.time.Duration; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -256,14 +257,16 @@ List listVolumes(String userName, String prefix, List getPendingDeletionKeys(int count) throws IOException; /** - * Returns the names of up to {@code count} open keys that are older than - * the configured expiration age. + * Returns the names of up to {@code count} open keys whose age is + * greater than or equal to {@code expireThreshold}. * * @param count The maximum number of open keys to return. + * @param expireThreshold The threshold of open key expire age. * @return a list of {@link String} representing names of open expired keys. * @throws IOException */ - List getExpiredOpenKeys(int count) throws IOException; + List getExpiredOpenKeys(Duration expireThreshold, int count) + throws IOException; /** * Returns the user Table. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index f073dce24008..f86eaf4758c9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import java.io.IOException; +import java.time.Duration; import java.util.List; /** @@ -117,15 +118,17 @@ List listTrash(String volumeName, String bucketName, List getPendingDeletionKeys(int count) throws IOException; /** - * Returns the names of up to {@code count} open keys that are older than - * the configured expiration age. + * Returns the names of up to {@code count} open keys whose age is + * greater than or equal to {@code expireThreshold}. * * @param count The maximum number of expired open keys to return. + * @param expireThreshold The threshold of open key expiration age. * @return a list of {@link String} representing the names of expired * open keys. * @throws IOException */ - List getExpiredOpenKeys(int count) throws IOException; + List getExpiredOpenKeys(Duration expireThreshold, int count) + throws IOException; /** * Returns the metadataManager. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 666d1d9aa8ff..03e9361ac509 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -21,6 +21,7 @@ import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -130,6 +131,7 @@ import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; import static org.apache.hadoop.util.Time.monotonicNow; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -577,8 +579,9 @@ public List getPendingDeletionKeys(final int count) } @Override - public List getExpiredOpenKeys(int count) throws IOException { - return metadataManager.getExpiredOpenKeys(count); + public List getExpiredOpenKeys(Duration expireThreshold, + int count) throws IOException { + return metadataManager.getExpiredOpenKeys(expireThreshold, count); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index c2dcb20f9ec9..a92418640626 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -21,7 +21,6 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -82,8 +81,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.DB_TRANSIENT_MARKER; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; @@ -195,7 +192,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager { private DBStore store; private final OzoneManagerLock lock; - private final long openKeyExpireThresholdMS; private Table userTable; private Table volumeTable; @@ -230,9 +226,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager { public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException { this.lock = new OzoneManagerLock(conf); - this.openKeyExpireThresholdMS = 1000L * conf.getInt( - OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, - OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT); // TODO: This is a temporary check. Once fully implemented, all OM state // change should go through Ratis - be it standalone (for non-HA) or // replicated (for HA). @@ -252,8 +245,6 @@ public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException { protected OmMetadataManagerImpl() { OzoneConfiguration conf = new OzoneConfiguration(); this.lock = new OzoneManagerLock(conf); - this.openKeyExpireThresholdMS = - OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT; this.omEpoch = 0; } @@ -1124,29 +1115,27 @@ public List getPendingDeletionKeys(final int keyCount) } @Override - public List getExpiredOpenKeys(int count) throws IOException { + public List getExpiredOpenKeys(Duration expireThreshold, + int count) throws IOException { // Only check for expired keys in the open key table, not its cache. // If a key expires while it is in the cache, it will be cleaned // up after the cache is flushed. - final Duration expirationDuration = - Duration.of(openKeyExpireThresholdMS, ChronoUnit.MILLIS); List expiredKeys = Lists.newArrayList(); - try ( - TableIterator> - keyValueTableIterator = getOpenKeyTable( - getBucketLayout()).iterator()) { + try (TableIterator> + keyValueTableIterator = getOpenKeyTable(getBucketLayout()).iterator()) { + + final long queryTime = Instant.now().toEpochMilli(); while (keyValueTableIterator.hasNext() && expiredKeys.size() < count) { KeyValue openKeyValue = keyValueTableIterator.next(); String openKey = openKeyValue.getKey(); OmKeyInfo openKeyInfo = openKeyValue.getValue(); - Duration openKeyAge = Duration - .between(Instant.ofEpochMilli(openKeyInfo.getCreationTime()), - Instant.now()); + final long openKeyAgeMillis = queryTime - openKeyInfo.getCreationTime(); + final Duration openKeyAge = Duration.ofMillis(openKeyAgeMillis); - if (openKeyAge.compareTo(expirationDuration) >= 0) { + if (openKeyAge.compareTo(expireThreshold) >= 0) { expiredKeys.add(openKey); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java index b0f19fcacd29..05dec6c3b882 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.time.Duration; import java.util.concurrent.TimeUnit; /** @@ -77,7 +78,7 @@ public BackgroundTaskResult call() throws Exception { // The new API for deleting expired open keys in OM HA will differ // significantly from the old implementation. // The old implementation has been removed so the code compiles. - keyManager.getExpiredOpenKeys(0); + keyManager.getExpiredOpenKeys(Duration.ZERO, 0); } catch (IOException e) { LOG.error("Unable to get hanging open keys, retry in" + " next interval", e); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 7354a940baa5..f267a093bde7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -34,6 +34,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.time.Duration; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -41,9 +42,10 @@ import java.util.TreeSet; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; @@ -343,12 +345,10 @@ public void testListKeys() throws Exception { TreeSet keysCSet = new TreeSet<>(); for (int i = 1; i <= 100; i++) { if (i % 2 == 0) { - keysASet.add( - prefixKeyA + i); + keysASet.add(prefixKeyA + i); addKeysToOM(volumeNameA, ozoneBucket, prefixKeyA + i, i); } else { - keysBSet.add( - prefixKeyB + i); + keysBSet.add(prefixKeyB + i); addKeysToOM(volumeNameA, hadoopBucket, prefixKeyB + i, i); } } @@ -553,12 +553,15 @@ public void testGetExpiredOpenKeys() throws Exception { final long clientID = 1000L; // To create expired keys, they will be assigned a creation time twice as // old as the minimum expiration time. - final long minExpiredTimeSeconds = ozoneConfiguration.getInt( - OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, - OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT); - final long expiredAgeMillis = - Instant.now().minus(minExpiredTimeSeconds * 2, - ChronoUnit.SECONDS).toEpochMilli(); + final long expireThresholdMillis = ozoneConfiguration.getTimeDuration( + OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, + OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD_DEFAULT, + TimeUnit.MILLISECONDS); + + final Duration expireThreshold = Duration.ofMillis(expireThresholdMillis); + + final long expiredOpenKeyCreationTime = Instant.now() + .minus(expireThresholdMillis * 2, ChronoUnit.MILLIS).toEpochMilli(); // Add expired keys to open key table. // The method under test does not check for expired open keys in the @@ -566,30 +569,31 @@ public void testGetExpiredOpenKeys() throws Exception { Set expiredKeys = new HashSet<>(); for (int i = 0; i < numExpiredOpenKeys; i++) { OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, "expired" + i, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE, 0L, expiredAgeMillis); + bucketName, "expired" + i, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L, expiredOpenKeyCreationTime); OMRequestTestUtils.addKeyToTable(true, false, - keyInfo, clientID, 0L, omMetadataManager); + keyInfo, clientID, 0L, omMetadataManager); String groupID = omMetadataManager.getOpenKey(volumeName, bucketName, - keyInfo.getKeyName(), clientID); + keyInfo.getKeyName(), clientID); expiredKeys.add(groupID); } // Add unexpired keys to open key table. for (int i = 0; i < numUnexpiredOpenKeys; i++) { OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, "unexpired" + i, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE); + bucketName, "unexpired" + i, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE); OMRequestTestUtils.addKeyToTable(true, false, - keyInfo, clientID, 0L, omMetadataManager); + keyInfo, clientID, 0L, omMetadataManager); } // Test retrieving fewer expired keys than actually exist. List someExpiredKeys = - omMetadataManager.getExpiredOpenKeys(numExpiredOpenKeys - 1); + omMetadataManager.getExpiredOpenKeys(expireThreshold, + numExpiredOpenKeys - 1); Assert.assertEquals(numExpiredOpenKeys - 1, someExpiredKeys.size()); for (String key: someExpiredKeys) { @@ -598,7 +602,8 @@ public void testGetExpiredOpenKeys() throws Exception { // Test attempting to retrieving more expired keys than actually exist. List allExpiredKeys = - omMetadataManager.getExpiredOpenKeys(numExpiredOpenKeys + 1); + omMetadataManager.getExpiredOpenKeys(expireThreshold, + numExpiredOpenKeys + 1); Assert.assertEquals(numExpiredOpenKeys, allExpiredKeys.size()); for (String key: allExpiredKeys) { @@ -607,7 +612,8 @@ public void testGetExpiredOpenKeys() throws Exception { // Test retrieving exact amount of expired keys that exist. allExpiredKeys = - omMetadataManager.getExpiredOpenKeys(numExpiredOpenKeys); + omMetadataManager.getExpiredOpenKeys(expireThreshold, + numExpiredOpenKeys); Assert.assertEquals(numExpiredOpenKeys, allExpiredKeys.size()); for (String key: allExpiredKeys) {