From 832c0b912358a7795329db9d3fb00fb949ea8df5 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 19 Apr 2022 11:35:07 +0800 Subject: [PATCH 01/13] Add parentID in OpenKeyBucket --- .../interface-client/src/main/proto/OmClientProtocol.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 45a191d2de41..6470e238218e 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1027,6 +1027,7 @@ message OpenKeyBucket { message OpenKey { required string name = 1; required uint64 clientID = 2; + optional uint64 parentID = 3; } message OMTokenProto { From 87b2c2dbcb5b02556f9b408d34ab76be944c8466 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 24 Mar 2022 22:28:12 +0800 Subject: [PATCH 02/13] Return List in getExpiredOpenKeys --- .../hadoop/ozone/om/OMMetadataManager.java | 7 +-- .../apache/hadoop/ozone/om/KeyManager.java | 8 ++-- .../hadoop/ozone/om/KeyManagerImpl.java | 7 +-- .../ozone/om/OmMetadataManagerImpl.java | 44 +++++++++++++------ .../ozone/om/TestOmMetadataManager.java | 40 ++++++++++------- 5 files changed, 68 insertions(+), 38 deletions(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index aee7d78957b0..cba438842ab8 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import org.apache.hadoop.ozone.storage.proto. OzoneManagerStorageProtos.PersistedUserVolumeInfo; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; @@ -260,12 +261,12 @@ List listVolumes(String userName, String prefix, * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. * - * @param count The maximum number of open keys to return. + * @param limit The maximum number of open keys to return. * @param expireThreshold The threshold of open key expire age. - * @return a list of {@link String} representing names of open expired keys. + * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ - List getExpiredOpenKeys(Duration expireThreshold, int count) + List getExpiredOpenKeys(Duration expireThreshold, int limit) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index f86eaf4758c9..c808984d35fd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.om.fs.OzoneManagerFS; import org.apache.hadoop.hdds.utils.BackgroundService; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import java.io.IOException; import java.time.Duration; @@ -121,13 +122,12 @@ List listTrash(String volumeName, String bucketName, * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. * - * @param count The maximum number of expired open keys to return. + * @param limit The maximum number of expired open keys to return. * @param expireThreshold The threshold of open key expiration age. - * @return a list of {@link String} representing the names of expired - * open keys. + * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ - List getExpiredOpenKeys(Duration expireThreshold, int count) + List getExpiredOpenKeys(Duration expireThreshold, int limit) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 124b2722e813..485527716d77 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -86,6 +86,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -582,9 +583,9 @@ public List getPendingDeletionKeys(final int count) } @Override - public List getExpiredOpenKeys(Duration expireThreshold, - int count) throws IOException { - return metadataManager.getExpiredOpenKeys(expireThreshold, count); + public List getExpiredOpenKeys(Duration expireThreshold, + int limit) throws IOException { + return metadataManager.getExpiredOpenKeys(expireThreshold, limit); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index b609b26f0de9..12cbac5d99be 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -73,6 +73,8 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKey; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import org.apache.hadoop.ozone.storage.proto .OzoneManagerStorageProtos.PersistedUserVolumeInfo; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; @@ -1193,33 +1195,49 @@ public List getPendingDeletionKeys(final int keyCount) } @Override - public List getExpiredOpenKeys(Duration expireThreshold, - int count) throws IOException { + public List getExpiredOpenKeys(Duration expireThreshold, + int limit) throws IOException { + Map expiredKeys = new HashMap<>(); + // Only check for expired keys in the open key table, not its cache. // If a key expires while it is in the cache, it will be cleaned // up after the cache is flushed. - List expiredKeys = Lists.newArrayList(); - try (TableIterator> keyValueTableIterator = getOpenKeyTable(getBucketLayout()).iterator()) { - final long queryTime = Instant.now().toEpochMilli(); + final long expiredCreationTimestamp = + Instant.now().minus(expireThreshold).toEpochMilli(); + + OpenKey.Builder builder = OpenKey.newBuilder(); - while (keyValueTableIterator.hasNext() && expiredKeys.size() < count) { + int count = 0; + while (count < limit && keyValueTableIterator.hasNext()) { KeyValue openKeyValue = keyValueTableIterator.next(); - String openKey = openKeyValue.getKey(); + String dbOpenKeyName = openKeyValue.getKey(); + long clientID = Long.parseLong(dbOpenKeyName.substring( + dbOpenKeyName.lastIndexOf(OM_KEY_PREFIX) + 1)); OmKeyInfo openKeyInfo = openKeyValue.getValue(); - final long openKeyAgeMillis = queryTime - openKeyInfo.getCreationTime(); - final Duration openKeyAge = Duration.ofMillis(openKeyAgeMillis); - - if (openKeyAge.compareTo(expireThreshold) >= 0) { - expiredKeys.add(openKey); + if (openKeyInfo.getCreationTime() <= expiredCreationTimestamp) { + final String volume = openKeyInfo.getVolumeName(); + final String bucket = openKeyInfo.getBucketName(); + final String mapKey = volume + OM_KEY_PREFIX + bucket; + if (!expiredKeys.containsKey(mapKey)) { + expiredKeys.put(mapKey, + OpenKeyBucket.newBuilder() + .setVolumeName(volume) + .setBucketName(bucket)); + } + builder.setName(openKeyInfo.getKeyName()) + .setClientID(clientID); + expiredKeys.get(mapKey).addKeys(builder.build()); + count++; } } } - return expiredKeys; + return expiredKeys.values().stream().map(OpenKeyBucket.Builder::build) + .collect(Collectors.toList()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index f267a093bde7..299094221fe6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -28,6 +28,8 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKey; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -35,6 +37,7 @@ import org.junit.rules.TemporaryFolder; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -591,34 +594,41 @@ public void testGetExpiredOpenKeys() throws Exception { } // Test retrieving fewer expired keys than actually exist. - List someExpiredKeys = + List someExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, numExpiredOpenKeys - 1); - - Assert.assertEquals(numExpiredOpenKeys - 1, someExpiredKeys.size()); - for (String key: someExpiredKeys) { - Assert.assertTrue(expiredKeys.contains(key)); - } + List names = flatten(someExpiredKeys); + Assert.assertEquals(numExpiredOpenKeys - 1, names.size()); + Assert.assertTrue(expiredKeys.containsAll(names)); // Test attempting to retrieving more expired keys than actually exist. - List allExpiredKeys = + List allExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, numExpiredOpenKeys + 1); - - Assert.assertEquals(numExpiredOpenKeys, allExpiredKeys.size()); - for (String key: allExpiredKeys) { - Assert.assertTrue(expiredKeys.contains(key)); - } + names = flatten(allExpiredKeys); + Assert.assertEquals(numExpiredOpenKeys, names.size()); + Assert.assertTrue(expiredKeys.containsAll(names)); // Test retrieving exact amount of expired keys that exist. allExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, numExpiredOpenKeys); + names = flatten(allExpiredKeys); + Assert.assertEquals(numExpiredOpenKeys, names.size()); + Assert.assertTrue(expiredKeys.containsAll(names)); + } - Assert.assertEquals(numExpiredOpenKeys, allExpiredKeys.size()); - for (String key: allExpiredKeys) { - Assert.assertTrue(expiredKeys.contains(key)); + private List flatten(List openKeyBuckets) { + List openKeyNames = new ArrayList<>(); + for (OpenKeyBucket openKeyBucket: openKeyBuckets) { + for (OpenKey openKey : openKeyBucket.getKeysList()) { + final String openKeyName = omMetadataManager.getOpenKey( + openKeyBucket.getVolumeName(), openKeyBucket.getBucketName(), + openKey.getName(), openKey.getClientID()); + openKeyNames.add(openKeyName); + } } + return openKeyNames; } private void addKeysToOM(String volumeName, String bucketName, From 7c5b0939eb1df891a8cac436a7bbf379922f731b Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 19 Apr 2022 14:30:38 +0800 Subject: [PATCH 03/13] Support FSO buckets in getExpiredOpenKeys --- .../hadoop/ozone/om/OMMetadataManager.java | 5 +- .../apache/hadoop/ozone/om/KeyManager.java | 6 +- .../hadoop/ozone/om/KeyManagerImpl.java | 5 +- .../ozone/om/OmMetadataManagerImpl.java | 14 ++- .../ozone/om/OpenKeyCleanupService.java | 3 +- .../ozone/om/TestOmMetadataManager.java | 85 ++++++++++++------- 6 files changed, 77 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index cba438842ab8..c0588fecef7e 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -263,11 +263,12 @@ List listVolumes(String userName, String prefix, * * @param limit The maximum number of open keys to return. * @param expireThreshold The threshold of open key expire age. + * @param bucketLayout The type of open keys to get (e.g. DEFAULT or FSO). * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ - List getExpiredOpenKeys(Duration expireThreshold, int limit) - throws IOException; + List getExpiredOpenKeys(Duration expireThreshold, int limit, + BucketLayout bucketLayout) throws IOException; /** * Returns the user Table. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index c808984d35fd..f19247e9b58d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; @@ -124,11 +125,12 @@ List listTrash(String volumeName, String bucketName, * * @param limit The maximum number of expired open keys to return. * @param expireThreshold The threshold of open key expiration age. + * @param bucketLayout The type of open keys to get (e.g. DEFAULT or FSO). * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ - List getExpiredOpenKeys(Duration expireThreshold, int limit) - throws IOException; + List getExpiredOpenKeys(Duration expireThreshold, int limit, + BucketLayout bucketLayout) throws IOException; /** * Returns the metadataManager. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 485527716d77..e866915f6dfb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -584,8 +584,9 @@ public List getPendingDeletionKeys(final int count) @Override public List getExpiredOpenKeys(Duration expireThreshold, - int limit) throws IOException { - return metadataManager.getExpiredOpenKeys(expireThreshold, limit); + int limit, BucketLayout bucketLayout) throws IOException { + return metadataManager.getExpiredOpenKeys(expireThreshold, limit, + bucketLayout); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 12cbac5d99be..1c84211e4c92 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1196,14 +1196,14 @@ public List getPendingDeletionKeys(final int keyCount) @Override public List getExpiredOpenKeys(Duration expireThreshold, - int limit) throws IOException { + int limit, BucketLayout bucketLayout) throws IOException { Map expiredKeys = new HashMap<>(); // Only check for expired keys in the open key table, not its cache. // If a key expires while it is in the cache, it will be cleaned // up after the cache is flushed. try (TableIterator> - keyValueTableIterator = getOpenKeyTable(getBucketLayout()).iterator()) { + keyValueTableIterator = getOpenKeyTable(bucketLayout).iterator()) { final long expiredCreationTimestamp = Instant.now().minus(expireThreshold).toEpochMilli(); @@ -1228,8 +1228,14 @@ public List getExpiredOpenKeys(Duration expireThreshold, .setVolumeName(volume) .setBucketName(bucket)); } - builder.setName(openKeyInfo.getKeyName()) - .setClientID(clientID); + if (bucketLayout.isFileSystemOptimized()) { + builder.setParentID(openKeyInfo.getParentObjectID()) + .setName(openKeyInfo.getFileName()) + .setClientID(clientID); + } else { + builder.setName(openKeyInfo.getKeyName()) + .setClientID(clientID); + } expiredKeys.get(mapKey).addKeys(builder.build()); count++; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java index 05dec6c3b882..1a4e17c96fdb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OpenKeyCleanupService.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +79,7 @@ public BackgroundTaskResult call() throws Exception { // The new API for deleting expired open keys in OM HA will differ // significantly from the old implementation. // The old implementation has been removed so the code compiles. - keyManager.getExpiredOpenKeys(Duration.ZERO, 0); + keyManager.getExpiredOpenKeys(Duration.ZERO, 0, BucketLayout.DEFAULT); } catch (IOException e) { LOG.error("Unable to get hanging open keys, retry in" + " next interval", e); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 299094221fe6..a568990cbdad 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKey; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; @@ -44,7 +45,7 @@ import java.util.Set; import java.util.TreeSet; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; @@ -549,12 +550,22 @@ private static BucketLayout getDefaultBucketLayout() { @Test public void testGetExpiredOpenKeys() throws Exception { - final String bucketName = "bucket"; - final String volumeName = "volume"; + testGetExpiredOpenKeys(BucketLayout.DEFAULT); + } + + @Test + public void testGetExpiredOpenKeysFSO() throws Exception { + testGetExpiredOpenKeys(BucketLayout.FILE_SYSTEM_OPTIMIZED); + } + + private void testGetExpiredOpenKeys(BucketLayout bucketLayout) + throws Exception { + final String bucketName = UUID.randomUUID().toString(); + final String volumeName = UUID.randomUUID().toString(); final int numExpiredOpenKeys = 4; final int numUnexpiredOpenKeys = 1; final long clientID = 1000L; - // To create expired keys, they will be assigned a creation time twice as + // To create expired keys, they will be assigned a creation time as // old as the minimum expiration time. final long expireThresholdMillis = ozoneConfiguration.getTimeDuration( OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD, @@ -563,40 +574,48 @@ public void testGetExpiredOpenKeys() throws Exception { final Duration expireThreshold = Duration.ofMillis(expireThresholdMillis); - final long expiredOpenKeyCreationTime = Instant.now() - .minus(expireThresholdMillis * 2, ChronoUnit.MILLIS).toEpochMilli(); + final long expiredOpenKeyCreationTime = + Instant.now().minus(expireThreshold).toEpochMilli(); // Add expired keys to open key table. // The method under test does not check for expired open keys in the // cache, since they will be picked up once the cache is flushed. Set expiredKeys = new HashSet<>(); - for (int i = 0; i < numExpiredOpenKeys; i++) { - OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, "expired" + i, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE, 0L, expiredOpenKeyCreationTime); - - OMRequestTestUtils.addKeyToTable(true, false, - keyInfo, clientID, 0L, omMetadataManager); + for (int i = 0; i < numExpiredOpenKeys + numUnexpiredOpenKeys; i++) { + OmKeyInfo keyInfo; - String groupID = omMetadataManager.getOpenKey(volumeName, bucketName, - keyInfo.getKeyName(), clientID); - expiredKeys.add(groupID); - } + if (i < numExpiredOpenKeys) { + keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, "expired" + i, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L, expiredOpenKeyCreationTime); + } else { + keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, "unexpired" + i, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE); + } - // Add unexpired keys to open key table. - for (int i = 0; i < numUnexpiredOpenKeys; i++) { - OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, "unexpired" + i, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE); + String dbOpenKeyName; - OMRequestTestUtils.addKeyToTable(true, false, - keyInfo, clientID, 0L, omMetadataManager); + if (bucketLayout.isFileSystemOptimized()) { + keyInfo.setParentObjectID(i); + keyInfo.setFileName(OzoneFSUtils.getFileName(keyInfo.getKeyName())); + OMRequestTestUtils.addFileToKeyTable(true, false, + keyInfo.getFileName(), keyInfo, clientID, 0L, omMetadataManager); + dbOpenKeyName = omMetadataManager.getOpenFileName( + keyInfo.getParentObjectID(), keyInfo.getFileName(), clientID); + } else { + OMRequestTestUtils.addKeyToTable(true, false, + keyInfo, clientID, 0L, omMetadataManager); + dbOpenKeyName = omMetadataManager.getOpenKey(volumeName, bucketName, + keyInfo.getKeyName(), clientID); + } + expiredKeys.add(dbOpenKeyName); } // Test retrieving fewer expired keys than actually exist. List someExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, - numExpiredOpenKeys - 1); + numExpiredOpenKeys - 1, bucketLayout); List names = flatten(someExpiredKeys); Assert.assertEquals(numExpiredOpenKeys - 1, names.size()); Assert.assertTrue(expiredKeys.containsAll(names)); @@ -604,7 +623,7 @@ public void testGetExpiredOpenKeys() throws Exception { // Test attempting to retrieving more expired keys than actually exist. List allExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, - numExpiredOpenKeys + 1); + numExpiredOpenKeys + 1, bucketLayout); names = flatten(allExpiredKeys); Assert.assertEquals(numExpiredOpenKeys, names.size()); Assert.assertTrue(expiredKeys.containsAll(names)); @@ -612,7 +631,7 @@ public void testGetExpiredOpenKeys() throws Exception { // Test retrieving exact amount of expired keys that exist. allExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, - numExpiredOpenKeys); + numExpiredOpenKeys, bucketLayout); names = flatten(allExpiredKeys); Assert.assertEquals(numExpiredOpenKeys, names.size()); Assert.assertTrue(expiredKeys.containsAll(names)); @@ -622,9 +641,15 @@ private List flatten(List openKeyBuckets) { List openKeyNames = new ArrayList<>(); for (OpenKeyBucket openKeyBucket: openKeyBuckets) { for (OpenKey openKey : openKeyBucket.getKeysList()) { - final String openKeyName = omMetadataManager.getOpenKey( - openKeyBucket.getVolumeName(), openKeyBucket.getBucketName(), - openKey.getName(), openKey.getClientID()); + final String openKeyName; + if (openKey.hasParentID()) { + openKeyName = omMetadataManager.getOpenFileName( + openKey.getParentID(), openKey.getName(), openKey.getClientID()); + } else { + openKeyName = omMetadataManager.getOpenKey( + openKeyBucket.getVolumeName(), openKeyBucket.getBucketName(), + openKey.getName(), openKey.getClientID()); + } openKeyNames.add(openKeyName); } } From 6035f79b542a4b2f27ac0cde41f9d22407d2bef1 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 29 Apr 2022 10:41:05 +0800 Subject: [PATCH 04/13] Update javadoc Co-authored-by: Ritesh H Shukla --- .../main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index c0588fecef7e..59c899e45166 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -264,7 +264,7 @@ List listVolumes(String userName, String prefix, * @param limit The maximum number of open keys to return. * @param expireThreshold The threshold of open key expire age. * @param bucketLayout The type of open keys to get (e.g. DEFAULT or FSO). - * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. + * @return a {@link List} of {@link OpenKeyBucket}: List of open keys per bucket. * @throws IOException */ List getExpiredOpenKeys(Duration expireThreshold, int limit, From 6ed94267b111eb40653f6b7d6f5639ff59810587 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 29 Apr 2022 11:21:15 +0800 Subject: [PATCH 05/13] Revert "Update javadoc" This reverts commit 6035f79b542a4b2f27ac0cde41f9d22407d2bef1. --- .../main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 59c899e45166..c0588fecef7e 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -264,7 +264,7 @@ List listVolumes(String userName, String prefix, * @param limit The maximum number of open keys to return. * @param expireThreshold The threshold of open key expire age. * @param bucketLayout The type of open keys to get (e.g. DEFAULT or FSO). - * @return a {@link List} of {@link OpenKeyBucket}: List of open keys per bucket. + * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ List getExpiredOpenKeys(Duration expireThreshold, int limit, From 2bfea4cd45ed294ae2330931d8ce61e092157700 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 29 Apr 2022 11:27:22 +0800 Subject: [PATCH 06/13] Add static method getClientID() --- .../apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1c84211e4c92..b8f7032fb9e4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -587,6 +587,12 @@ public String getOpenKey(String volume, String bucket, return openKey; } + private static long getClientID(String dbOpenKeyName) { + int beginIndex = dbOpenKeyName.lastIndexOf(OM_KEY_PREFIX) + 1; + String clientID = dbOpenKeyName.substring(beginIndex); + return Long.parseLong(clientID); + } + @Override public String getMultipartKey(String volume, String bucket, String key, String @@ -1214,8 +1220,7 @@ public List getExpiredOpenKeys(Duration expireThreshold, while (count < limit && keyValueTableIterator.hasNext()) { KeyValue openKeyValue = keyValueTableIterator.next(); String dbOpenKeyName = openKeyValue.getKey(); - long clientID = Long.parseLong(dbOpenKeyName.substring( - dbOpenKeyName.lastIndexOf(OM_KEY_PREFIX) + 1)); + long clientID = getClientID(dbOpenKeyName); OmKeyInfo openKeyInfo = openKeyValue.getValue(); if (openKeyInfo.getCreationTime() <= expiredCreationTimestamp) { From 00cc3fb680b478639e5634b48c36646e2b0d020e Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 6 May 2022 09:55:33 +0800 Subject: [PATCH 07/13] Use OpenKey.name as dbOpenKeyName --- .../src/main/proto/OmClientProtocol.proto | 4 ++-- .../ozone/om/OmMetadataManagerImpl.java | 21 ++++-------------- .../ozone/om/TestOmMetadataManager.java | 22 +++++-------------- 3 files changed, 11 insertions(+), 36 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 34b70a7aeff5..baba7376a963 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1027,12 +1027,12 @@ message OpenKeyBucket { required string volumeName = 1; required string bucketName = 2; repeated OpenKey keys = 3; + optional BucketLayoutProto bucketLayout = 4; // TODO: change to required } message OpenKey { required string name = 1; - required uint64 clientID = 2; - optional uint64 parentID = 3; + optional uint64 clientID = 2; // TODO: remove } message OMTokenProto { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index b8f7032fb9e4..bcb2a1e51d5a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -587,12 +587,6 @@ public String getOpenKey(String volume, String bucket, return openKey; } - private static long getClientID(String dbOpenKeyName) { - int beginIndex = dbOpenKeyName.lastIndexOf(OM_KEY_PREFIX) + 1; - String clientID = dbOpenKeyName.substring(beginIndex); - return Long.parseLong(clientID); - } - @Override public String getMultipartKey(String volume, String bucket, String key, String @@ -1220,7 +1214,6 @@ public List getExpiredOpenKeys(Duration expireThreshold, while (count < limit && keyValueTableIterator.hasNext()) { KeyValue openKeyValue = keyValueTableIterator.next(); String dbOpenKeyName = openKeyValue.getKey(); - long clientID = getClientID(dbOpenKeyName); OmKeyInfo openKeyInfo = openKeyValue.getValue(); if (openKeyInfo.getCreationTime() <= expiredCreationTimestamp) { @@ -1231,17 +1224,11 @@ public List getExpiredOpenKeys(Duration expireThreshold, expiredKeys.put(mapKey, OpenKeyBucket.newBuilder() .setVolumeName(volume) - .setBucketName(bucket)); - } - if (bucketLayout.isFileSystemOptimized()) { - builder.setParentID(openKeyInfo.getParentObjectID()) - .setName(openKeyInfo.getFileName()) - .setClientID(clientID); - } else { - builder.setName(openKeyInfo.getKeyName()) - .setClientID(clientID); + .setBucketName(bucket) + .setBucketLayout(bucketLayout.toProto())); } - expiredKeys.get(mapKey).addKeys(builder.build()); + expiredKeys.get(mapKey) + .addKeys(builder.setName(dbOpenKeyName).build()); count++; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index a568990cbdad..77afbbae59c6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -38,7 +38,6 @@ import org.junit.rules.TemporaryFolder; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -47,6 +46,7 @@ import java.time.Instant; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD_DEFAULT; @@ -638,22 +638,10 @@ private void testGetExpiredOpenKeys(BucketLayout bucketLayout) } private List flatten(List openKeyBuckets) { - List openKeyNames = new ArrayList<>(); - for (OpenKeyBucket openKeyBucket: openKeyBuckets) { - for (OpenKey openKey : openKeyBucket.getKeysList()) { - final String openKeyName; - if (openKey.hasParentID()) { - openKeyName = omMetadataManager.getOpenFileName( - openKey.getParentID(), openKey.getName(), openKey.getClientID()); - } else { - openKeyName = omMetadataManager.getOpenKey( - openKeyBucket.getVolumeName(), openKeyBucket.getBucketName(), - openKey.getName(), openKey.getClientID()); - } - openKeyNames.add(openKeyName); - } - } - return openKeyNames; + return openKeyBuckets.stream() + .flatMap(openKeyBucket -> openKeyBucket.getKeysList().stream()) + .map(OpenKey::getName) + .collect(Collectors.toList()); } private void addKeysToOM(String volumeName, String bucketName, From 6964f5d2a1a2cd2961b5835868fca00b2f542d20 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 6 May 2022 17:30:02 +0800 Subject: [PATCH 08/13] Update comment to trigger CI --- .../interface-client/src/main/proto/OmClientProtocol.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index baba7376a963..d89af14da88e 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1032,7 +1032,7 @@ message OpenKeyBucket { message OpenKey { required string name = 1; - optional uint64 clientID = 2; // TODO: remove + optional uint64 clientID = 2; // TODO: to be removed } message OMTokenProto { From ad78f0679374446bcf80d7c824c7b0d6629cdba6 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 6 May 2022 10:48:42 +0800 Subject: [PATCH 09/13] Change flatten -> getOpenKeyNames --- .../apache/hadoop/ozone/om/TestOmMetadataManager.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 77afbbae59c6..6d8f51eb93f6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -616,7 +616,7 @@ private void testGetExpiredOpenKeys(BucketLayout bucketLayout) List someExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, numExpiredOpenKeys - 1, bucketLayout); - List names = flatten(someExpiredKeys); + List names = getOpenKeyNames(someExpiredKeys); Assert.assertEquals(numExpiredOpenKeys - 1, names.size()); Assert.assertTrue(expiredKeys.containsAll(names)); @@ -624,7 +624,7 @@ private void testGetExpiredOpenKeys(BucketLayout bucketLayout) List allExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, numExpiredOpenKeys + 1, bucketLayout); - names = flatten(allExpiredKeys); + names = getOpenKeyNames(allExpiredKeys); Assert.assertEquals(numExpiredOpenKeys, names.size()); Assert.assertTrue(expiredKeys.containsAll(names)); @@ -632,14 +632,15 @@ private void testGetExpiredOpenKeys(BucketLayout bucketLayout) allExpiredKeys = omMetadataManager.getExpiredOpenKeys(expireThreshold, numExpiredOpenKeys, bucketLayout); - names = flatten(allExpiredKeys); + names = getOpenKeyNames(allExpiredKeys); Assert.assertEquals(numExpiredOpenKeys, names.size()); Assert.assertTrue(expiredKeys.containsAll(names)); } - private List flatten(List openKeyBuckets) { + private List getOpenKeyNames(List openKeyBuckets) { return openKeyBuckets.stream() - .flatMap(openKeyBucket -> openKeyBucket.getKeysList().stream()) + .map(OpenKeyBucket::getKeysList) + .flatMap(List::stream) .map(OpenKey::getName) .collect(Collectors.toList()); } From 295d1e64dc46dd293abe22831b08aa3502541234 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Sat, 7 May 2022 11:02:13 +0800 Subject: [PATCH 10/13] Resolve TODO in proto --- .../interface-client/src/main/proto/OmClientProtocol.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index d89af14da88e..48583cad8153 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1027,12 +1027,12 @@ message OpenKeyBucket { required string volumeName = 1; required string bucketName = 2; repeated OpenKey keys = 3; - optional BucketLayoutProto bucketLayout = 4; // TODO: change to required + optional BucketLayoutProto bucketLayout = 4; } message OpenKey { required string name = 1; - optional uint64 clientID = 2; // TODO: to be removed + optional uint64 clientID = 2 [deprecated=true]; } message OMTokenProto { From dedc3179dd99d39ecf7d8c29ec67c545799b58e4 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Sat, 7 May 2022 11:11:42 +0800 Subject: [PATCH 11/13] Remove BucketLayout in OpenKeyBucket --- .../interface-client/src/main/proto/OmClientProtocol.proto | 1 - .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 48583cad8153..d91d8519a967 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1027,7 +1027,6 @@ message OpenKeyBucket { required string volumeName = 1; required string bucketName = 2; repeated OpenKey keys = 3; - optional BucketLayoutProto bucketLayout = 4; } message OpenKey { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index bcb2a1e51d5a..02e598613714 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1224,8 +1224,7 @@ public List getExpiredOpenKeys(Duration expireThreshold, expiredKeys.put(mapKey, OpenKeyBucket.newBuilder() .setVolumeName(volume) - .setBucketName(bucket) - .setBucketLayout(bucketLayout.toProto())); + .setBucketName(bucket)); } expiredKeys.get(mapKey) .addKeys(builder.setName(dbOpenKeyName).build()); From b0a60fff777ce980543b17e92f4316a67e363220 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 19 May 2022 11:16:25 +0800 Subject: [PATCH 12/13] Revert limit to count --- .../org/apache/hadoop/ozone/om/OMMetadataManager.java | 4 ++-- .../main/java/org/apache/hadoop/ozone/om/KeyManager.java | 4 ++-- .../java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 4 ++-- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index c0588fecef7e..9064968db327 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -261,13 +261,13 @@ List listVolumes(String userName, String prefix, * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. * - * @param limit The maximum number of open keys to return. + * @param count The maximum number of open keys to return. * @param expireThreshold The threshold of open key expire age. * @param bucketLayout The type of open keys to get (e.g. DEFAULT or FSO). * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ - List getExpiredOpenKeys(Duration expireThreshold, int limit, + List getExpiredOpenKeys(Duration expireThreshold, int count, BucketLayout bucketLayout) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index f19247e9b58d..9e5ae1cc494d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -123,13 +123,13 @@ List listTrash(String volumeName, String bucketName, * Returns the names of up to {@code count} open keys whose age is * greater than or equal to {@code expireThreshold}. * - * @param limit The maximum number of expired open keys to return. + * @param count The maximum number of expired open keys to return. * @param expireThreshold The threshold of open key expiration age. * @param bucketLayout The type of open keys to get (e.g. DEFAULT or FSO). * @return a {@link List} of {@link OpenKeyBucket}, the expired open keys. * @throws IOException */ - List getExpiredOpenKeys(Duration expireThreshold, int limit, + List getExpiredOpenKeys(Duration expireThreshold, int count, BucketLayout bucketLayout) throws IOException; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 9855fe15f2d7..8411e3e4c7f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -584,8 +584,8 @@ public List getPendingDeletionKeys(final int count) @Override public List getExpiredOpenKeys(Duration expireThreshold, - int limit, BucketLayout bucketLayout) throws IOException { - return metadataManager.getExpiredOpenKeys(expireThreshold, limit, + int count, BucketLayout bucketLayout) throws IOException { + return metadataManager.getExpiredOpenKeys(expireThreshold, count, bucketLayout); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 02e598613714..b25f3f82c4d3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1196,7 +1196,7 @@ public List getPendingDeletionKeys(final int keyCount) @Override public List getExpiredOpenKeys(Duration expireThreshold, - int limit, BucketLayout bucketLayout) throws IOException { + int count, BucketLayout bucketLayout) throws IOException { Map expiredKeys = new HashMap<>(); // Only check for expired keys in the open key table, not its cache. @@ -1210,8 +1210,8 @@ public List getExpiredOpenKeys(Duration expireThreshold, OpenKey.Builder builder = OpenKey.newBuilder(); - int count = 0; - while (count < limit && keyValueTableIterator.hasNext()) { + int num = 0; + while (num < count && keyValueTableIterator.hasNext()) { KeyValue openKeyValue = keyValueTableIterator.next(); String dbOpenKeyName = openKeyValue.getKey(); OmKeyInfo openKeyInfo = openKeyValue.getValue(); @@ -1228,7 +1228,7 @@ public List getExpiredOpenKeys(Duration expireThreshold, } expiredKeys.get(mapKey) .addKeys(builder.setName(dbOpenKeyName).build()); - count++; + num++; } } } From e20da1fc3e593a5be564a06cae1375b282f1f154 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 19 May 2022 14:30:08 +0800 Subject: [PATCH 13/13] Cleanup code in test --- .../ozone/om/TestOmMetadataManager.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 6d8f51eb93f6..aa8e98c91d21 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -582,20 +582,13 @@ private void testGetExpiredOpenKeys(BucketLayout bucketLayout) // cache, since they will be picked up once the cache is flushed. Set expiredKeys = new HashSet<>(); for (int i = 0; i < numExpiredOpenKeys + numUnexpiredOpenKeys; i++) { - OmKeyInfo keyInfo; - - if (i < numExpiredOpenKeys) { - keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, "expired" + i, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE, 0L, expiredOpenKeyCreationTime); - } else { - keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, - bucketName, "unexpired" + i, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.ONE); - } - - String dbOpenKeyName; + final long creationTime = i < numExpiredOpenKeys ? + expiredOpenKeyCreationTime : Instant.now().toEpochMilli(); + final OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, + bucketName, "expired" + i, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L, creationTime); + final String dbOpenKeyName; if (bucketLayout.isFileSystemOptimized()) { keyInfo.setParentObjectID(i); keyInfo.setFileName(OzoneFSUtils.getFileName(keyInfo.getKeyName()));