From ce7394d3c6bd475be7584c4a6b7bbf4119cb4bfd Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 13:38:45 +0200 Subject: [PATCH 1/8] test --- .../dist/src/main/smoketest/ec/basic.robot | 26 ++++++++++++++++--- .../hadoop/ozone/om/TestListKeysWithFSO.java | 6 +++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot index b2e7a78ff1ab..2c8c0ff892c9 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot @@ -28,11 +28,13 @@ ${VOLUME} vol${PREFIX} *** Keywords *** Create Key In EC Bucket [arguments] ${size} - ${key} = Set Variable /${VOLUME}/ecbucket/${size} + ${dir} = Set Variable /${VOLUME}/ecbucket/dir + ${key} = Set Variable ${dir}/${size} ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} Key Should Match Local File ${key} ${file} Verify Key EC Replication Config ${key} RS 3 2 1048576 + Verify Key EC Replication Config ${dir} RS 3 2 1048576 Get Disk Usage of File with EC RS Replication [arguments] ${fileLength} ${dataChunkCount} ${parityChunkCount} ${ecChunkSize} @@ -82,29 +84,45 @@ Create 3MB Key In EC Bucket Create 100MB Key In EC Bucket Create Key In EC Bucket 100mb +Create Key in Default Bucket + ${size} = Set Variable 1mb + ${dir} = Set Variable /${VOLUME}/default/dir + ${key} = Set Variable ${dir}/${size} + ${file} = Set Variable /tmp/${size} + Create Key ${key} ${file} + Key Should Match Local File ${key} ${file} + Verify Key Replica Replication Config ${key} RATIS THREE + Verify Key Replica Replication Config ${dir} RATIS THREE + Create Key in Ratis Bucket ${size} = Set Variable 1mb - ${key} = Set Variable /${VOLUME}/default/${size} + ${dir} = Set Variable /${VOLUME}/ratis/dir + ${key} = Set Variable ${dir}/${size} ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} Key Should Match Local File ${key} ${file} Verify Key Replica Replication Config ${key} RATIS THREE + Verify Key Replica Replication Config ${dir} RATIS THREE Create Ratis Key In EC Bucket ${size} = Set Variable 1mb - ${key} = Set Variable /${VOLUME}/ecbucket/${size}Ratis + ${dir} = Set Variable /${VOLUME}/ecbucket/dir2 + ${key} = Set Variable ${dir}/${size}Ratis ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} --replication=THREE --type=RATIS Key Should Match Local File ${key} ${file} Verify Key Replica Replication Config ${key} RATIS THREE + Verify Key EC Replication Config ${dir} RS 3 2 1048576 Create EC Key In Ratis Bucket ${size} = Set Variable 1mb - ${key} = Set Variable /${VOLUME}/ratis/${size}EC + ${dir} = Set Variable /${VOLUME}/ratis/dir2 + ${key} = Set Variable ${dir}/${size}EC ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} --replication=rs-3-2-1024k --type=EC Key Should Match Local File ${key} ${file} Verify Key EC Replication Config ${key} RS 3 2 1048576 + Verify Key Replica Replication Config ${dir} RATIS THREE Test Invalid Replication Parameters ${message} = Execute And Ignore Error ozone sh bucket create --replication=rs-3-2-1024k --type=RATIS /${VOLUME}/foo diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 53f1bc699eb0..5b6edace4df1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -44,6 +45,7 @@ import java.util.LinkedList; import java.util.List; import java.util.ArrayList; +import java.util.Optional; import java.util.UUID; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE; @@ -493,10 +495,14 @@ private void checkKeyList(String keyPrefix, String startKey, Iterator ozoneKeyIterator = fsoBucket.listKeys(keyPrefix, startKey); + ReplicationConfig expectedReplication = + Optional.ofNullable(fsoBucket.getReplicationConfig()) + .orElse(cluster.getOzoneManager().getDefaultReplicationConfig()); List keyLists = new ArrayList<>(); while (ozoneKeyIterator.hasNext()) { OzoneKey ozoneKey = ozoneKeyIterator.next(); + Assert.assertEquals(expectedReplication, ozoneKey.getReplicationConfig()); keyLists.add(ozoneKey.getName()); } LinkedList outputKeysList = new LinkedList(keyLists); From 23aa8651f845ce589d123996d543487bfc17b909 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 13:39:28 +0200 Subject: [PATCH 2/8] HDDS-8543. New key prefix in EC bucket with FSO reports RATIS replication --- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 6 ++++ .../hadoop/ozone/om/KeyManagerImpl.java | 32 ++++++++++++------- .../ozone/om/OzoneListStatusHelper.java | 31 ++++++++++++++---- .../ozone/om/request/file/OMFileRequest.java | 19 ++++++++--- 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index e48cf98e906c..b5eabf0575c5 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -421,6 +421,12 @@ public void setReplicationConfig(ReplicationConfig repConfig) { this.replicationConfig = repConfig; } + public void setReplicationIfMissing(ReplicationConfig repConfig) { + if (replicationConfig == null) { + replicationConfig = repConfig; + } + } + public FileChecksum getFileChecksum() { return fileChecksum; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ee15405cbde8..cba99a257c9e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -135,6 +135,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.addTrailingSlashIfNeeded; import static org.apache.hadoop.util.MetricUtil.captureLatencyNs; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; @@ -456,13 +457,17 @@ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, return null; } // Appended trailing slash to represent directory to the user + OmKeyInfo keyInfo = fileStatus.getKeyInfo(); if (fileStatus.isDirectory()) { - String keyPath = OzoneFSUtils.addTrailingSlashIfNeeded( - fileStatus.getKeyInfo().getKeyName()); - fileStatus.getKeyInfo().setKeyName(keyPath); + keyInfo.setKeyName(addTrailingSlashIfNeeded(keyInfo.getKeyName())); + fallbackToGlobalDefaultReplicationIfNeeded(keyInfo); } - fileStatus.getKeyInfo().setFile(fileStatus.isFile()); - return fileStatus.getKeyInfo(); + keyInfo.setFile(fileStatus.isFile()); + return keyInfo; + } + + private void fallbackToGlobalDefaultReplicationIfNeeded(OmKeyInfo keyInfo) { + keyInfo.setReplicationIfMissing(ozoneManager.getDefaultReplicationConfig()); } private void addBlockToken4Read(OmKeyInfo value) throws IOException { @@ -1130,7 +1135,7 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, BucketLayout layout = getBucketLayout(metadataManager, volumeName, bucketName); fileKeyInfo = metadataManager.getKeyTable(layout).get(fileKeyBytes); - String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + String dirKey = addTrailingSlashIfNeeded(keyName); // Check if the key is a directory. if (fileKeyInfo == null) { @@ -1200,8 +1205,8 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, */ private OmKeyInfo createFakeDirIfShould(String volume, String bucket, String keyName, BucketLayout layout) throws IOException { - String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); - String targetKey = OzoneFSUtils.addTrailingSlashIfNeeded( + String dirKey = addTrailingSlashIfNeeded(keyName); + String targetKey = addTrailingSlashIfNeeded( metadataManager.getOzoneKey(volume, bucket, keyName)); Table keyTable = metadataManager.getKeyTable(layout); @@ -1291,6 +1296,7 @@ private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, } return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); } else { + fallbackToGlobalDefaultReplicationIfNeeded(fileStatus.getKeyInfo()); return fileStatus; } } @@ -1318,7 +1324,7 @@ private OmKeyInfo createDirectoryKey(OmKeyInfo keyInfo, String keyName) OmBucketInfo bucketInfo = getBucketInfo(keyInfo.getVolumeName(), keyInfo.getBucketName()); - String dir = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + String dir = addTrailingSlashIfNeeded(keyName); FileEncryptionInfo encInfo = getFileEncryptionInfo(bucketInfo); return new OmKeyInfo.Builder() .setVolumeName(keyInfo.getVolumeName()) @@ -1477,7 +1483,8 @@ public List listStatus(OmKeyArgs args, boolean recursive, Preconditions.checkArgument(!recursive); OzoneListStatusHelper statusHelper = new OzoneListStatusHelper(metadataManager, scmBlockSize, - this::getOzoneFileStatusFSO); + this::getOzoneFileStatusFSO, + ozoneManager.getDefaultReplicationConfig()); Collection statuses = statusHelper.listStatusFSO(args, startKey, numEntries, clientAddress, allowPartialPrefixes); @@ -1496,11 +1503,11 @@ public List listStatus(OmKeyArgs args, boolean recursive, return Collections.singletonList(fileStatus); } // keyName is a directory - startKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + startKey = addTrailingSlashIfNeeded(keyName); } // Note: eliminating the case where startCacheKey could end with '//' - String keyArgs = OzoneFSUtils.addTrailingSlashIfNeeded( + String keyArgs = addTrailingSlashIfNeeded( metadataManager.getOzoneKey(volumeName, bucketName, keyName)); TableIterator> iterator; @@ -1856,6 +1863,7 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo( parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo, dirName); + // not setting replication for directories, only used for key deletion directories.add(omKeyInfo); countEntries++; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index c7093a46bad4..9d56bb6a5f2e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -19,6 +19,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -42,6 +44,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; +import java.util.Optional; import java.util.TreeMap; import java.util.Map; import java.util.PriorityQueue; @@ -81,12 +84,18 @@ public interface ClosableIterator extends Iterator, Closeable { private final OMMetadataManager metadataManager; private final long scmBlockSize; private final GetFileStatusHelper getStatusHelper; - - OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize, - GetFileStatusHelper func) { + private final ReplicationConfig omDefaultReplication; + + OzoneListStatusHelper( + OMMetadataManager metadataManager, + long scmBlockSize, + GetFileStatusHelper func, + ReplicationConfig omDefaultReplication + ) { this.metadataManager = metadataManager; this.scmBlockSize = scmBlockSize; this.getStatusHelper = func; + this.omDefaultReplication = omDefaultReplication; } public Collection listStatusFSO(OmKeyArgs args, @@ -191,6 +200,10 @@ public Collection listStatusFSO(OmKeyArgs args, TreeMap map = new TreeMap<>(); BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); + ReplicationConfig replication = + Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig()) + .map(DefaultReplicationConfig::getReplicationConfig) + .orElse(omDefaultReplication); // fetch the sorted output using a min heap iterator where // every remove from the heap will give the smallest entry. @@ -200,7 +213,7 @@ public Collection listStatusFSO(OmKeyArgs args, while (map.size() < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); OzoneFileStatus status = entry.getStatus(prefixKey, - scmBlockSize, volumeName, bucketName); + scmBlockSize, volumeName, bucketName, replication); map.put(entry.key, status); } } @@ -302,8 +315,13 @@ public int hashCode() { return key.hashCode(); } - public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, - String volumeName, String bucketName) { + public OzoneFileStatus getStatus( + String prefixPath, + long scmBlockSize, + String volumeName, + String bucketName, + ReplicationConfig bucketReplication + ) { OmKeyInfo keyInfo; if (entryType.isDir()) { Preconditions.checkArgument(value instanceof OmDirectoryInfo); @@ -312,6 +330,7 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, dirInfo.getName()); keyInfo = OMFileRequest.getOmKeyInfo(volumeName, bucketName, dirInfo, dirName); + keyInfo.setReplicationIfMissing(bucketReplication); } else { Preconditions.checkArgument(value instanceof OmKeyInfo); keyInfo = (OmKeyInfo) value; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 99ff0e72869a..9fcf1cb5c1b4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -26,11 +26,10 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -47,6 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.ratis.util.Preconditions; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -618,6 +618,8 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, /** * Gets OmKeyInfo if exists for the given key name in the DB. + * If the key is a directory, its replication config is set from the bucket's + * default replication config, if any. * * @param omMetadataMgr metadata manager * @param volumeName volume name @@ -675,6 +677,7 @@ public static OzoneFileStatus getOMKeyInfoIfExists( if (omDirInfo != null) { OmKeyInfo omKeyInfo = getOmKeyInfo(volumeName, bucketName, omDirInfo, keyName); + setDirectoryReplicationFromBucket(omBucketInfo, omKeyInfo); return new OzoneFileStatus(omKeyInfo, scmBlockSize, true); } @@ -682,8 +685,18 @@ public static OzoneFileStatus getOMKeyInfoIfExists( return null; } + private static void setDirectoryReplicationFromBucket( + OmBucketInfo bucket, + OmKeyInfo dir + ) { + Preconditions.assertTrue(!dir.isFile(), "expect only directory"); + Optional.ofNullable(bucket.getDefaultReplicationConfig()) + .ifPresent(r -> dir.setReplicationIfMissing(r.getReplicationConfig())); + } + /** * Prepare OmKeyInfo from OmDirectoryInfo. + * Replication config is not set, callers have to set it if needed. * * @param volumeName volume name * @param bucketName bucket name @@ -707,8 +720,6 @@ public static OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, builder.setObjectID(dirInfo.getObjectID()); builder.setUpdateID(dirInfo.getUpdateID()); builder.setFileName(dirInfo.getName()); - builder.setReplicationConfig(RatisReplicationConfig - .getInstance(HddsProtos.ReplicationFactor.ONE)); builder.setOmKeyLocationInfos(Collections.singletonList( new OmKeyLocationInfoGroup(0, new ArrayList<>()))); return builder.build(); From d64126264fba16b0182da90ba1c42a9a206fca61 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 15:27:35 +0200 Subject: [PATCH 3/8] add fake replication config for dir deletion --- .../java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 6 +++++- .../om/response/key/TestOMKeysDeleteResponseWithFSO.java | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index cba99a257c9e..1e9006f0e9f0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -46,9 +46,11 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo; @@ -1863,7 +1865,9 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo( parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo, dirName); - // not setting replication for directories, only used for key deletion + // add fake replication config for directories being deleted + omKeyInfo.setReplicationIfMissing( + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); directories.add(omKeyInfo); countEntries++; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java index bab036c621e2..c8e4558e7955 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.response.key; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -80,6 +81,9 @@ protected void createPreRequisities() throws Exception { OmKeyInfo dirKeyInfo = OMFileRequest.getOmKeyInfo(volumeName, bucketName, dirInfo, dir); + // add fake replication config for directories being deleted + dirKeyInfo.setReplicationIfMissing( + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); dirDeleteList.add(dirKeyInfo); dirDBKeys.add(dirOzoneDBKey); dirDelDBKeys.add(omMetadataManager.getOzoneDeletePathKey( From f185d293e26c35ef77f5a099088ac97bfd07a8f6 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 16:23:25 +0200 Subject: [PATCH 4/8] update key path in read.robot to match the ones created by basic.robot --- hadoop-ozone/dist/src/main/smoketest/ec/read.robot | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/read.robot b/hadoop-ozone/dist/src/main/smoketest/ec/read.robot index c51280631227..f9081ab9717a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/read.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ec/read.robot @@ -23,16 +23,16 @@ ${VOLUME} vol${PREFIX} *** Test Cases *** Read 1MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/1mb /tmp/1mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/1mb /tmp/1mb Read 2MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/2mb /tmp/2mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/2mb /tmp/2mb Read 3MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/3mb /tmp/3mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/3mb /tmp/3mb Read 100MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/100mb /tmp/100mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/100mb /tmp/100mb Read EC Key in Ratis Bucket - Key Should Match Local File /${VOLUME}/ratis/1mbEC /tmp/1mb + Key Should Match Local File /${VOLUME}/ratis/dir2/1mbEC /tmp/1mb From 730d29ee35f27d3bd7726c8d4962702d1fff509f Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 17:00:00 +0200 Subject: [PATCH 5/8] restore default replication config of RATIS/ONE --- .../hadoop/ozone/om/KeyManagerImpl.java | 11 +--- .../ozone/om/OzoneListStatusHelper.java | 2 +- .../ozone/om/request/file/OMFileRequest.java | 56 ++++++++++--------- .../key/OMKeyDeleteRequestWithFSO.java | 5 +- .../key/OMKeyRenameRequestWithFSO.java | 21 +++---- .../key/OMKeySetTimesRequestWithFSO.java | 4 +- .../om/request/key/OMKeysDeleteRequest.java | 20 +++---- .../key/OmKeysDeleteRequestWithFSO.java | 17 +++--- .../key/acl/OMKeyAclRequestWithFSO.java | 4 +- .../key/TestOMSetTimesRequestWithFSO.java | 4 +- .../key/TestOMKeysDeleteResponseWithFSO.java | 4 -- 11 files changed, 71 insertions(+), 77 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 1e9006f0e9f0..f3b4c4ce1dee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -46,11 +46,9 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo; @@ -453,7 +451,7 @@ private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, String keyName) throws IOException { OzoneFileStatus fileStatus = - OMFileRequest.getOMKeyInfoIfExists(metadataManager, + OMFileRequest.getOMKeyInfoIfExists(ozoneManager, metadataManager, volumeName, bucketName, keyName, scmBlockSize); if (fileStatus == null) { return null; @@ -1270,8 +1268,8 @@ private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, return new OzoneFileStatus(); } - fileStatus = OMFileRequest.getOMKeyInfoIfExists(metadataManager, - volumeName, bucketName, keyName, scmBlockSize); + fileStatus = OMFileRequest.getOMKeyInfoIfExists(ozoneManager, + metadataManager, volumeName, bucketName, keyName, scmBlockSize); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, @@ -1865,9 +1863,6 @@ private List gatherSubDirsWithIterator(OmKeyInfo parentInfo, OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo( parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo, dirName); - // add fake replication config for directories being deleted - omKeyInfo.setReplicationIfMissing( - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); directories.add(omKeyInfo); countEntries++; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 9d56bb6a5f2e..2c13e5cb5ddf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -330,7 +330,7 @@ public OzoneFileStatus getStatus( dirInfo.getName()); keyInfo = OMFileRequest.getOmKeyInfo(volumeName, bucketName, dirInfo, dirName); - keyInfo.setReplicationIfMissing(bucketReplication); + keyInfo.setReplicationConfig(bucketReplication); // always overwrite } else { Preconditions.checkArgument(value instanceof OmKeyInfo); keyInfo = (OmKeyInfo) value; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 9fcf1cb5c1b4..2cc314a3fc19 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -30,6 +30,10 @@ import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -38,6 +42,7 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; @@ -46,7 +51,6 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.ratis.util.Preconditions; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -621,18 +625,20 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, * If the key is a directory, its replication config is set from the bucket's * default replication config, if any. * + * @param ozoneManager * @param omMetadataMgr metadata manager - * @param volumeName volume name - * @param bucketName bucket name - * @param keyName key name - * @param scmBlockSize scm block size + * @param volumeName volume name + * @param bucketName bucket name + * @param keyName key name + * @param scmBlockSize scm block size * @return OzoneFileStatus * @throws IOException DB failure */ @Nullable public static OzoneFileStatus getOMKeyInfoIfExists( - OMMetadataManager omMetadataMgr, String volumeName, String bucketName, - String keyName, long scmBlockSize) throws IOException { + OzoneManager ozoneManager, OMMetadataManager omMetadataMgr, + String volumeName, String bucketName, String keyName, + long scmBlockSize) throws IOException { OMFileRequest.validateBucket(omMetadataMgr, volumeName, bucketName); @@ -677,7 +683,11 @@ public static OzoneFileStatus getOMKeyInfoIfExists( if (omDirInfo != null) { OmKeyInfo omKeyInfo = getOmKeyInfo(volumeName, bucketName, omDirInfo, keyName); - setDirectoryReplicationFromBucket(omBucketInfo, omKeyInfo); + ReplicationConfig replicationConfig = + Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig()) + .map(DefaultReplicationConfig::getReplicationConfig) + .orElse(ozoneManager.getDefaultReplicationConfig()); + omKeyInfo.setReplicationConfig(replicationConfig); return new OzoneFileStatus(omKeyInfo, scmBlockSize, true); } @@ -685,18 +695,8 @@ public static OzoneFileStatus getOMKeyInfoIfExists( return null; } - private static void setDirectoryReplicationFromBucket( - OmBucketInfo bucket, - OmKeyInfo dir - ) { - Preconditions.assertTrue(!dir.isFile(), "expect only directory"); - Optional.ofNullable(bucket.getDefaultReplicationConfig()) - .ifPresent(r -> dir.setReplicationIfMissing(r.getReplicationConfig())); - } - /** * Prepare OmKeyInfo from OmDirectoryInfo. - * Replication config is not set, callers have to set it if needed. * * @param volumeName volume name * @param bucketName bucket name @@ -720,6 +720,8 @@ public static OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, builder.setObjectID(dirInfo.getObjectID()); builder.setUpdateID(dirInfo.getUpdateID()); builder.setFileName(dirInfo.getName()); + builder.setReplicationConfig(RatisReplicationConfig + .getInstance(HddsProtos.ReplicationFactor.ONE)); builder.setOmKeyLocationInfos(Collections.singletonList( new OmKeyLocationInfoGroup(0, new ArrayList<>()))); return builder.build(); @@ -796,24 +798,26 @@ public static void verifyToDirIsASubDirOfFromDirectory(String fromKeyName, * Check whether dst parent dir exists or not. If the parent exists, then the * source can be renamed to dst path. * - * @param volumeName volume name - * @param bucketName bucket name - * @param toKeyName destination path - * @param metaMgr metadata manager + * @param volumeName volume name + * @param bucketName bucket name + * @param toKeyName destination path + * @param ozoneManager + * @param metaMgr metadata manager * @return omDirectoryInfo object of destination path's parent * or null if parent is bucket * @throws IOException if the destination parent is not a directory. */ - public static OmKeyInfo getKeyParentDir(String volumeName, String bucketName, - String toKeyName, OMMetadataManager metaMgr) throws IOException { + public static OmKeyInfo getKeyParentDir( + String volumeName, String bucketName, String toKeyName, + OzoneManager ozoneManager, OMMetadataManager metaMgr) throws IOException { int totalDirsCount = OzoneFSUtils.getFileCount(toKeyName); // skip parent is root '/' if (totalDirsCount <= 1) { return null; } String toKeyParentDir = OzoneFSUtils.getParentDir(toKeyName); - OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists(metaMgr, - volumeName, bucketName, toKeyParentDir, 0); + OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists( + ozoneManager, metaMgr, volumeName, bucketName, toKeyParentDir, 0); // check if the immediate parent exists if (toKeyParentDirStatus == null) { throw new OMException(String.format( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index 892e17b7366a..2f41493c12b2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -108,9 +108,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Validate bucket and volume exists or not. validateBucketAndVolume(omMetadataManager, volumeName, bucketName); - OzoneFileStatus keyStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName, - bucketName, keyName, 0); + OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( + ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0); if (keyStatus == null) { throw new OMException("Key not found. Key:" + keyName, KEY_NOT_FOUND); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java index 5fd9aa4650c6..36a0fa26e9e5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java @@ -135,9 +135,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, validateBucketAndVolume(omMetadataManager, volumeName, bucketName); // Check if fromKey exists - OzoneFileStatus fromKeyFileStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName, - bucketName, fromKeyName, 0); + OzoneFileStatus fromKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists( + ozoneManager, omMetadataManager, volumeName, bucketName, fromKeyName, + 0); // case-1) fromKeyName should exist, otw throws exception if (fromKeyFileStatus == null) { @@ -159,9 +159,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OMFileRequest.verifyToDirIsASubDirOfFromDirectory(fromKeyName, toKeyName, fromKeyFileStatus.isDirectory()); - OzoneFileStatus toKeyFileStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, - volumeName, bucketName, toKeyName, 0); + OzoneFileStatus toKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists( + ozoneManager, omMetadataManager, + volumeName, bucketName, toKeyName, 0); // Check if toKey exists. if (toKeyFileStatus != null) { @@ -186,8 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String newToKeyName = OzoneFSUtils.appendFileNameToKeyPath(toKeyName, fromFileName); OzoneFileStatus newToOzoneFileStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, - volumeName, bucketName, newToKeyName, 0); + OMFileRequest.getOMKeyInfoIfExists( + ozoneManager, omMetadataManager, + volumeName, bucketName, newToKeyName, 0); if (newToOzoneFileStatus != null) { // case-5) If new destin '/dst/source' exists then throws exception @@ -213,7 +214,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // doesn't exist then throw exception, otw the source can be renamed to // destination path. OmKeyInfo toKeyParent = OMFileRequest.getKeyParentDir(volumeName, - bucketName, toKeyName, omMetadataManager); + bucketName, toKeyName, ozoneManager, omMetadataManager); omClientResponse = renameKey(toKeyParent, toKeyName, fromKeyValue, fromKeyName, isRenameDirectory, keyArgs.getModificationTime(), @@ -297,7 +298,7 @@ private OMClientResponse renameKey(OmKeyInfo toKeyParent, String toKeyName, setModificationTime(ommm, omBucketInfo, toKeyParent, volumeId, bucketId, modificationTime, dirTable, trxnLogIndex); fromKeyParent = OMFileRequest.getKeyParentDir(fromKeyValue.getVolumeName(), - fromKeyValue.getBucketName(), fromKeyName, metadataMgr); + fromKeyValue.getBucketName(), fromKeyName, ozoneManager, metadataMgr); if (fromKeyParent == null && omBucketInfo == null) { // Get omBucketInfo only when needed to reduce unnecessary DB IO omBucketInfo = metadataMgr.getBucketTable().get(bucketKey); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java index fa9f7a20e34e..fa5477f78b26 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java @@ -89,8 +89,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } lockAcquired = omMetadataManager.getLock() .acquireWriteLock(BUCKET_LOCK, volume, bucket); - OzoneFileStatus keyStatus = OMFileRequest - .getOMKeyInfoIfExists(omMetadataManager, volume, bucket, key, 0); + OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( + ozoneManager, omMetadataManager, volume, bucket, key, 0); if (keyStatus == null) { throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 7636170b6cfb..8911c68bc7ee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -137,8 +137,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String keyName = deleteKeyArgs.getKeys(indexFailed); String objectKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); - OmKeyInfo omKeyInfo = - getOmKeyInfo(omMetadataManager, volumeName, bucketName, keyName); + OmKeyInfo omKeyInfo = getOmKeyInfo(ozoneManager, omMetadataManager, + volumeName, bucketName, keyName); if (omKeyInfo == null) { deleteStatus = false; @@ -154,9 +154,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY, volumeOwner); + OzoneFileStatus fileStatus = getOzoneKeyStatus( + ozoneManager, omMetadataManager, volumeName, bucketName, keyName); addKeyToAppropriateList(omKeyInfoList, omKeyInfo, dirList, - getOzoneKeyStatus(omMetadataManager, volumeName, bucketName, - keyName)); + fileStatus); } catch (Exception ex) { deleteStatus = false; LOG.error("Acl check failed for Key: {}", objectKey, ex); @@ -241,8 +242,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } protected OzoneFileStatus getOzoneKeyStatus( - OMMetadataManager omMetadataManager, String volumeName, String bucketName, - String keyName) throws IOException { + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, + String volumeName, String bucketName, String keyName) throws IOException { // implemented in child class return null; } @@ -287,12 +288,11 @@ protected void addKeyToAppropriateList(List omKeyInfoList, omKeyInfoList.add(omKeyInfo); } - protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager, + protected OmKeyInfo getOmKeyInfo( + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, String volume, String bucket, String key) throws IOException { String objectKey = omMetadataManager.getOzoneKey(volume, bucket, key); - OmKeyInfo omKeyInfo = - omMetadataManager.getKeyTable(getBucketLayout()).get(objectKey); - return omKeyInfo; + return omMetadataManager.getKeyTable(getBucketLayout()).get(objectKey); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index 11c2ef7e89f0..83d202269fec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -62,11 +62,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } @Override - protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager, + protected OmKeyInfo getOmKeyInfo( + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, String volumeName, String bucketName, String keyName) throws IOException { - OzoneFileStatus keyStatus = - getOzoneKeyStatus(omMetadataManager, volumeName, bucketName, keyName); + OzoneFileStatus keyStatus = getOzoneKeyStatus( + ozoneManager, omMetadataManager, volumeName, bucketName, keyName); return keyStatus != null ? keyStatus.getKeyInfo() : null; } @@ -82,12 +83,10 @@ protected void addKeyToAppropriateList(List omKeyInfoList, @Override protected OzoneFileStatus getOzoneKeyStatus( - OMMetadataManager omMetadataManager, String volumeName, String bucketName, - String keyName) throws IOException { - OzoneFileStatus keyStatus = OMFileRequest - .getOMKeyInfoIfExists(omMetadataManager, volumeName, bucketName, - keyName, 0); - return keyStatus; + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, + String volumeName, String bucketName, String keyName) throws IOException { + return OMFileRequest.getOMKeyInfoIfExists(ozoneManager, omMetadataManager, + volumeName, bucketName, keyName, 0); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java index e83486bb9072..d9bb2619d38b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java @@ -89,8 +89,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } lockAcquired = omMetadataManager.getLock() .acquireWriteLock(BUCKET_LOCK, volume, bucket); - OzoneFileStatus keyStatus = OMFileRequest - .getOMKeyInfoIfExists(omMetadataManager, volume, bucket, key, 0); + OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( + ozoneManager, omMetadataManager, volume, bucket, key, 0); if (keyStatus == null) { throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java index 93d48cb9e9fa..ba01d0b1bcaa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java @@ -54,7 +54,7 @@ public void testDirSetTimesRequest() throws Exception { long mtime = 2000; executeAndReturn(mtime); OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( - omMetadataManager, volumeName, bucketName, keyName, 0); + ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0); assertNotNull(keyStatus); assertTrue(keyStatus.isDirectory()); long keyMtime = keyStatus.getKeyInfo().getModificationTime(); @@ -63,7 +63,7 @@ public void testDirSetTimesRequest() throws Exception { long newMtime = -1; executeAndReturn(newMtime); keyStatus = OMFileRequest.getOMKeyInfoIfExists( - omMetadataManager, volumeName, bucketName, keyName, 0); + ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0); assertNotNull(keyStatus); assertTrue(keyStatus.isDirectory()); keyMtime = keyStatus.getKeyInfo().getModificationTime(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java index c8e4558e7955..bab036c621e2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.response.key; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -81,9 +80,6 @@ protected void createPreRequisities() throws Exception { OmKeyInfo dirKeyInfo = OMFileRequest.getOmKeyInfo(volumeName, bucketName, dirInfo, dir); - // add fake replication config for directories being deleted - dirKeyInfo.setReplicationIfMissing( - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)); dirDeleteList.add(dirKeyInfo); dirDBKeys.add(dirOzoneDBKey); dirDelDBKeys.add(omMetadataManager.getOzoneDeletePathKey( From a01dc96a5379da4cc34af52c826e71035481876d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 17:15:38 +0200 Subject: [PATCH 6/8] remove leftover code --- .../java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 6 ------ .../java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 6 ------ 2 files changed, 12 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index b5eabf0575c5..e48cf98e906c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -421,12 +421,6 @@ public void setReplicationConfig(ReplicationConfig repConfig) { this.replicationConfig = repConfig; } - public void setReplicationIfMissing(ReplicationConfig repConfig) { - if (replicationConfig == null) { - replicationConfig = repConfig; - } - } - public FileChecksum getFileChecksum() { return fileChecksum; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index f3b4c4ce1dee..3e2abd33252a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -460,16 +460,11 @@ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, OmKeyInfo keyInfo = fileStatus.getKeyInfo(); if (fileStatus.isDirectory()) { keyInfo.setKeyName(addTrailingSlashIfNeeded(keyInfo.getKeyName())); - fallbackToGlobalDefaultReplicationIfNeeded(keyInfo); } keyInfo.setFile(fileStatus.isFile()); return keyInfo; } - private void fallbackToGlobalDefaultReplicationIfNeeded(OmKeyInfo keyInfo) { - keyInfo.setReplicationIfMissing(ozoneManager.getDefaultReplicationConfig()); - } - private void addBlockToken4Read(OmKeyInfo value) throws IOException { Preconditions.checkNotNull(value, "OMKeyInfo cannot be null"); if (grpcBlockTokenEnabled) { @@ -1296,7 +1291,6 @@ private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, } return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); } else { - fallbackToGlobalDefaultReplicationIfNeeded(fileStatus.getKeyInfo()); return fileStatus; } } From fe6e0bb5d71810489803dc92e89b988ad78d3580 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 21 Jun 2023 17:18:30 +0200 Subject: [PATCH 7/8] reduce change --- .../hadoop/ozone/om/KeyManagerImpl.java | 22 +++++++++---------- .../ozone/om/request/file/OMFileRequest.java | 11 ++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 3e2abd33252a..7592d53b57e0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -135,7 +135,6 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.SCM_GET_PIPELINE_EXCEPTION; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.addTrailingSlashIfNeeded; import static org.apache.hadoop.util.MetricUtil.captureLatencyNs; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType.KEY; @@ -457,12 +456,13 @@ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, return null; } // Appended trailing slash to represent directory to the user - OmKeyInfo keyInfo = fileStatus.getKeyInfo(); if (fileStatus.isDirectory()) { - keyInfo.setKeyName(addTrailingSlashIfNeeded(keyInfo.getKeyName())); + String keyPath = OzoneFSUtils.addTrailingSlashIfNeeded( + fileStatus.getKeyInfo().getKeyName()); + fileStatus.getKeyInfo().setKeyName(keyPath); } - keyInfo.setFile(fileStatus.isFile()); - return keyInfo; + fileStatus.getKeyInfo().setFile(fileStatus.isFile()); + return fileStatus.getKeyInfo(); } private void addBlockToken4Read(OmKeyInfo value) throws IOException { @@ -1130,7 +1130,7 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, BucketLayout layout = getBucketLayout(metadataManager, volumeName, bucketName); fileKeyInfo = metadataManager.getKeyTable(layout).get(fileKeyBytes); - String dirKey = addTrailingSlashIfNeeded(keyName); + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); // Check if the key is a directory. if (fileKeyInfo == null) { @@ -1200,8 +1200,8 @@ private OzoneFileStatus getOzoneFileStatus(OmKeyArgs args, */ private OmKeyInfo createFakeDirIfShould(String volume, String bucket, String keyName, BucketLayout layout) throws IOException { - String dirKey = addTrailingSlashIfNeeded(keyName); - String targetKey = addTrailingSlashIfNeeded( + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + String targetKey = OzoneFSUtils.addTrailingSlashIfNeeded( metadataManager.getOzoneKey(volume, bucket, keyName)); Table keyTable = metadataManager.getKeyTable(layout); @@ -1318,7 +1318,7 @@ private OmKeyInfo createDirectoryKey(OmKeyInfo keyInfo, String keyName) OmBucketInfo bucketInfo = getBucketInfo(keyInfo.getVolumeName(), keyInfo.getBucketName()); - String dir = addTrailingSlashIfNeeded(keyName); + String dir = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); FileEncryptionInfo encInfo = getFileEncryptionInfo(bucketInfo); return new OmKeyInfo.Builder() .setVolumeName(keyInfo.getVolumeName()) @@ -1497,11 +1497,11 @@ public List listStatus(OmKeyArgs args, boolean recursive, return Collections.singletonList(fileStatus); } // keyName is a directory - startKey = addTrailingSlashIfNeeded(keyName); + startKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); } // Note: eliminating the case where startCacheKey could end with '//' - String keyArgs = addTrailingSlashIfNeeded( + String keyArgs = OzoneFSUtils.addTrailingSlashIfNeeded( metadataManager.getOzoneKey(volumeName, bucketName, keyName)); TableIterator> iterator; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 2cc314a3fc19..b96bb029e0a8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -622,15 +622,12 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, /** * Gets OmKeyInfo if exists for the given key name in the DB. - * If the key is a directory, its replication config is set from the bucket's - * default replication config, if any. * - * @param ozoneManager * @param omMetadataMgr metadata manager - * @param volumeName volume name - * @param bucketName bucket name - * @param keyName key name - * @param scmBlockSize scm block size + * @param volumeName volume name + * @param bucketName bucket name + * @param keyName key name + * @param scmBlockSize scm block size * @return OzoneFileStatus * @throws IOException DB failure */ From 2599d54255390abc3c19aaa0cb4d24f96c31f047 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 22 Jun 2023 08:41:24 +0200 Subject: [PATCH 8/8] pass ReplicationConfig instead of OzoneManager --- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 11 ++++++----- .../ozone/om/request/file/OMFileRequest.java | 10 ++++++---- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 3 ++- .../om/request/key/OMKeyRenameRequestWithFSO.java | 14 +++++++------- .../request/key/OMKeySetTimesRequestWithFSO.java | 3 ++- .../om/request/key/OmKeysDeleteRequestWithFSO.java | 5 +++-- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 3 ++- .../request/key/TestOMSetTimesRequestWithFSO.java | 6 ++++-- 8 files changed, 32 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 7592d53b57e0..46eda684dff1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -449,9 +449,9 @@ private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, */ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, String keyName) throws IOException { - OzoneFileStatus fileStatus = - OMFileRequest.getOMKeyInfoIfExists(ozoneManager, metadataManager, - volumeName, bucketName, keyName, scmBlockSize); + OzoneFileStatus fileStatus = OMFileRequest.getOMKeyInfoIfExists( + metadataManager, volumeName, bucketName, keyName, scmBlockSize, + ozoneManager.getDefaultReplicationConfig()); if (fileStatus == null) { return null; } @@ -1263,8 +1263,9 @@ private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, return new OzoneFileStatus(); } - fileStatus = OMFileRequest.getOMKeyInfoIfExists(ozoneManager, - metadataManager, volumeName, bucketName, keyName, scmBlockSize); + fileStatus = OMFileRequest.getOMKeyInfoIfExists(metadataManager, + volumeName, bucketName, keyName, scmBlockSize, + ozoneManager.getDefaultReplicationConfig()); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index b96bb029e0a8..17565fec5d0c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -633,9 +633,10 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, */ @Nullable public static OzoneFileStatus getOMKeyInfoIfExists( - OzoneManager ozoneManager, OMMetadataManager omMetadataMgr, + OMMetadataManager omMetadataMgr, String volumeName, String bucketName, String keyName, - long scmBlockSize) throws IOException { + long scmBlockSize, ReplicationConfig defaultReplication + ) throws IOException { OMFileRequest.validateBucket(omMetadataMgr, volumeName, bucketName); @@ -683,7 +684,7 @@ public static OzoneFileStatus getOMKeyInfoIfExists( ReplicationConfig replicationConfig = Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig()) .map(DefaultReplicationConfig::getReplicationConfig) - .orElse(ozoneManager.getDefaultReplicationConfig()); + .orElse(defaultReplication); omKeyInfo.setReplicationConfig(replicationConfig); return new OzoneFileStatus(omKeyInfo, scmBlockSize, true); } @@ -814,7 +815,8 @@ public static OmKeyInfo getKeyParentDir( } String toKeyParentDir = OzoneFSUtils.getParentDir(toKeyName); OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists( - ozoneManager, metaMgr, volumeName, bucketName, toKeyParentDir, 0); + metaMgr, volumeName, bucketName, toKeyParentDir, 0, + ozoneManager.getDefaultReplicationConfig()); // check if the immediate parent exists if (toKeyParentDirStatus == null) { throw new OMException(String.format( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index 2f41493c12b2..9a2ddeb72624 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -109,7 +109,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, validateBucketAndVolume(omMetadataManager, volumeName, bucketName); OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0); + omMetadataManager, volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); if (keyStatus == null) { throw new OMException("Key not found. Key:" + keyName, KEY_NOT_FOUND); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java index 36a0fa26e9e5..1fbfdfa34103 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java @@ -136,8 +136,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Check if fromKey exists OzoneFileStatus fromKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, volumeName, bucketName, fromKeyName, - 0); + omMetadataManager, volumeName, bucketName, fromKeyName, 0, + ozoneManager.getDefaultReplicationConfig()); // case-1) fromKeyName should exist, otw throws exception if (fromKeyFileStatus == null) { @@ -160,8 +160,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, toKeyName, fromKeyFileStatus.isDirectory()); OzoneFileStatus toKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, - volumeName, bucketName, toKeyName, 0); + omMetadataManager, volumeName, bucketName, toKeyName, 0, + ozoneManager.getDefaultReplicationConfig()); // Check if toKey exists. if (toKeyFileStatus != null) { @@ -186,9 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String newToKeyName = OzoneFSUtils.appendFileNameToKeyPath(toKeyName, fromFileName); OzoneFileStatus newToOzoneFileStatus = - OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, - volumeName, bucketName, newToKeyName, 0); + OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, + volumeName, bucketName, newToKeyName, 0, + ozoneManager.getDefaultReplicationConfig()); if (newToOzoneFileStatus != null) { // case-5) If new destin '/dst/source' exists then throws exception diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java index fa5477f78b26..6fb29cb4bc1b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java @@ -90,7 +90,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, lockAcquired = omMetadataManager.getLock() .acquireWriteLock(BUCKET_LOCK, volume, bucket); OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, volume, bucket, key, 0); + omMetadataManager, volume, bucket, key, 0, + ozoneManager.getDefaultReplicationConfig()); if (keyStatus == null) { throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index 83d202269fec..7fe527bd95ea 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -85,8 +85,9 @@ protected void addKeyToAppropriateList(List omKeyInfoList, protected OzoneFileStatus getOzoneKeyStatus( OzoneManager ozoneManager, OMMetadataManager omMetadataManager, String volumeName, String bucketName, String keyName) throws IOException { - return OMFileRequest.getOMKeyInfoIfExists(ozoneManager, omMetadataManager, - volumeName, bucketName, keyName, 0); + return OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, + volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java index d9bb2619d38b..ebd4c5dd8db4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java @@ -90,7 +90,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, lockAcquired = omMetadataManager.getLock() .acquireWriteLock(BUCKET_LOCK, volume, bucket); OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, volume, bucket, key, 0); + omMetadataManager, volume, bucket, key, 0, + ozoneManager.getDefaultReplicationConfig()); if (keyStatus == null) { throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java index ba01d0b1bcaa..1ad5a0931df3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java @@ -54,7 +54,8 @@ public void testDirSetTimesRequest() throws Exception { long mtime = 2000; executeAndReturn(mtime); OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0); + omMetadataManager, volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); assertNotNull(keyStatus); assertTrue(keyStatus.isDirectory()); long keyMtime = keyStatus.getKeyInfo().getModificationTime(); @@ -63,7 +64,8 @@ public void testDirSetTimesRequest() throws Exception { long newMtime = -1; executeAndReturn(newMtime); keyStatus = OMFileRequest.getOMKeyInfoIfExists( - ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0); + omMetadataManager, volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); assertNotNull(keyStatus); assertTrue(keyStatus.isDirectory()); keyMtime = keyStatus.getKeyInfo().getModificationTime();