diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot index b2e7a78ff1ab..2c8c0ff892c9 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot @@ -28,11 +28,13 @@ ${VOLUME} vol${PREFIX} *** Keywords *** Create Key In EC Bucket [arguments] ${size} - ${key} = Set Variable /${VOLUME}/ecbucket/${size} + ${dir} = Set Variable /${VOLUME}/ecbucket/dir + ${key} = Set Variable ${dir}/${size} ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} Key Should Match Local File ${key} ${file} Verify Key EC Replication Config ${key} RS 3 2 1048576 + Verify Key EC Replication Config ${dir} RS 3 2 1048576 Get Disk Usage of File with EC RS Replication [arguments] ${fileLength} ${dataChunkCount} ${parityChunkCount} ${ecChunkSize} @@ -82,29 +84,45 @@ Create 3MB Key In EC Bucket Create 100MB Key In EC Bucket Create Key In EC Bucket 100mb +Create Key in Default Bucket + ${size} = Set Variable 1mb + ${dir} = Set Variable /${VOLUME}/default/dir + ${key} = Set Variable ${dir}/${size} + ${file} = Set Variable /tmp/${size} + Create Key ${key} ${file} + Key Should Match Local File ${key} ${file} + Verify Key Replica Replication Config ${key} RATIS THREE + Verify Key Replica Replication Config ${dir} RATIS THREE + Create Key in Ratis Bucket ${size} = Set Variable 1mb - ${key} = Set Variable /${VOLUME}/default/${size} + ${dir} = Set Variable /${VOLUME}/ratis/dir + ${key} = Set Variable ${dir}/${size} ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} Key Should Match Local File ${key} ${file} Verify Key Replica Replication Config ${key} RATIS THREE + Verify Key Replica Replication Config ${dir} RATIS THREE Create Ratis Key In EC Bucket ${size} = Set Variable 1mb - ${key} = Set Variable /${VOLUME}/ecbucket/${size}Ratis + ${dir} = Set Variable /${VOLUME}/ecbucket/dir2 + ${key} = Set Variable ${dir}/${size}Ratis ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} --replication=THREE --type=RATIS Key Should Match Local File ${key} ${file} Verify Key Replica Replication Config ${key} RATIS THREE + Verify Key EC Replication Config ${dir} RS 3 2 1048576 Create EC Key In Ratis Bucket ${size} = Set Variable 1mb - ${key} = Set Variable /${VOLUME}/ratis/${size}EC + ${dir} = Set Variable /${VOLUME}/ratis/dir2 + ${key} = Set Variable ${dir}/${size}EC ${file} = Set Variable /tmp/${size} Create Key ${key} ${file} --replication=rs-3-2-1024k --type=EC Key Should Match Local File ${key} ${file} Verify Key EC Replication Config ${key} RS 3 2 1048576 + Verify Key Replica Replication Config ${dir} RATIS THREE Test Invalid Replication Parameters ${message} = Execute And Ignore Error ozone sh bucket create --replication=rs-3-2-1024k --type=RATIS /${VOLUME}/foo diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/read.robot b/hadoop-ozone/dist/src/main/smoketest/ec/read.robot index c51280631227..f9081ab9717a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/read.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ec/read.robot @@ -23,16 +23,16 @@ ${VOLUME} vol${PREFIX} *** Test Cases *** Read 1MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/1mb /tmp/1mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/1mb /tmp/1mb Read 2MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/2mb /tmp/2mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/2mb /tmp/2mb Read 3MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/3mb /tmp/3mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/3mb /tmp/3mb Read 100MB EC Key - Key Should Match Local File /${VOLUME}/ecbucket/100mb /tmp/100mb + Key Should Match Local File /${VOLUME}/ecbucket/dir/100mb /tmp/100mb Read EC Key in Ratis Bucket - Key Should Match Local File /${VOLUME}/ratis/1mbEC /tmp/1mb + Key Should Match Local File /${VOLUME}/ratis/dir2/1mbEC /tmp/1mb diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 53f1bc699eb0..5b6edace4df1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -44,6 +45,7 @@ import java.util.LinkedList; import java.util.List; import java.util.ArrayList; +import java.util.Optional; import java.util.UUID; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE; @@ -493,10 +495,14 @@ private void checkKeyList(String keyPrefix, String startKey, Iterator ozoneKeyIterator = fsoBucket.listKeys(keyPrefix, startKey); + ReplicationConfig expectedReplication = + Optional.ofNullable(fsoBucket.getReplicationConfig()) + .orElse(cluster.getOzoneManager().getDefaultReplicationConfig()); List keyLists = new ArrayList<>(); while (ozoneKeyIterator.hasNext()) { OzoneKey ozoneKey = ozoneKeyIterator.next(); + Assert.assertEquals(expectedReplication, ozoneKey.getReplicationConfig()); keyLists.add(ozoneKey.getName()); } LinkedList outputKeysList = new LinkedList(keyLists); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ee15405cbde8..46eda684dff1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -449,9 +449,9 @@ private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, */ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, String keyName) throws IOException { - OzoneFileStatus fileStatus = - OMFileRequest.getOMKeyInfoIfExists(metadataManager, - volumeName, bucketName, keyName, scmBlockSize); + OzoneFileStatus fileStatus = OMFileRequest.getOMKeyInfoIfExists( + metadataManager, volumeName, bucketName, keyName, scmBlockSize, + ozoneManager.getDefaultReplicationConfig()); if (fileStatus == null) { return null; } @@ -1264,7 +1264,8 @@ private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args, } fileStatus = OMFileRequest.getOMKeyInfoIfExists(metadataManager, - volumeName, bucketName, keyName, scmBlockSize); + volumeName, bucketName, keyName, scmBlockSize, + ozoneManager.getDefaultReplicationConfig()); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, @@ -1477,7 +1478,8 @@ public List listStatus(OmKeyArgs args, boolean recursive, Preconditions.checkArgument(!recursive); OzoneListStatusHelper statusHelper = new OzoneListStatusHelper(metadataManager, scmBlockSize, - this::getOzoneFileStatusFSO); + this::getOzoneFileStatusFSO, + ozoneManager.getDefaultReplicationConfig()); Collection statuses = statusHelper.listStatusFSO(args, startKey, numEntries, clientAddress, allowPartialPrefixes); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index c7093a46bad4..2c13e5cb5ddf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -19,6 +19,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -42,6 +44,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; +import java.util.Optional; import java.util.TreeMap; import java.util.Map; import java.util.PriorityQueue; @@ -81,12 +84,18 @@ public interface ClosableIterator extends Iterator, Closeable { private final OMMetadataManager metadataManager; private final long scmBlockSize; private final GetFileStatusHelper getStatusHelper; - - OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize, - GetFileStatusHelper func) { + private final ReplicationConfig omDefaultReplication; + + OzoneListStatusHelper( + OMMetadataManager metadataManager, + long scmBlockSize, + GetFileStatusHelper func, + ReplicationConfig omDefaultReplication + ) { this.metadataManager = metadataManager; this.scmBlockSize = scmBlockSize; this.getStatusHelper = func; + this.omDefaultReplication = omDefaultReplication; } public Collection listStatusFSO(OmKeyArgs args, @@ -191,6 +200,10 @@ public Collection listStatusFSO(OmKeyArgs args, TreeMap map = new TreeMap<>(); BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); + ReplicationConfig replication = + Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig()) + .map(DefaultReplicationConfig::getReplicationConfig) + .orElse(omDefaultReplication); // fetch the sorted output using a min heap iterator where // every remove from the heap will give the smallest entry. @@ -200,7 +213,7 @@ public Collection listStatusFSO(OmKeyArgs args, while (map.size() < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); OzoneFileStatus status = entry.getStatus(prefixKey, - scmBlockSize, volumeName, bucketName); + scmBlockSize, volumeName, bucketName, replication); map.put(entry.key, status); } } @@ -302,8 +315,13 @@ public int hashCode() { return key.hashCode(); } - public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, - String volumeName, String bucketName) { + public OzoneFileStatus getStatus( + String prefixPath, + long scmBlockSize, + String volumeName, + String bucketName, + ReplicationConfig bucketReplication + ) { OmKeyInfo keyInfo; if (entryType.isDir()) { Preconditions.checkArgument(value instanceof OmDirectoryInfo); @@ -312,6 +330,7 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, dirInfo.getName()); keyInfo = OMFileRequest.getOmKeyInfo(volumeName, bucketName, dirInfo, dirName); + keyInfo.setReplicationConfig(bucketReplication); // always overwrite } else { Preconditions.checkArgument(value instanceof OmKeyInfo); keyInfo = (OmKeyInfo) value; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 99ff0e72869a..17565fec5d0c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -26,10 +26,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; @@ -39,6 +42,7 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; @@ -629,8 +633,10 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable, */ @Nullable public static OzoneFileStatus getOMKeyInfoIfExists( - OMMetadataManager omMetadataMgr, String volumeName, String bucketName, - String keyName, long scmBlockSize) throws IOException { + OMMetadataManager omMetadataMgr, + String volumeName, String bucketName, String keyName, + long scmBlockSize, ReplicationConfig defaultReplication + ) throws IOException { OMFileRequest.validateBucket(omMetadataMgr, volumeName, bucketName); @@ -675,6 +681,11 @@ public static OzoneFileStatus getOMKeyInfoIfExists( if (omDirInfo != null) { OmKeyInfo omKeyInfo = getOmKeyInfo(volumeName, bucketName, omDirInfo, keyName); + ReplicationConfig replicationConfig = + Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig()) + .map(DefaultReplicationConfig::getReplicationConfig) + .orElse(defaultReplication); + omKeyInfo.setReplicationConfig(replicationConfig); return new OzoneFileStatus(omKeyInfo, scmBlockSize, true); } @@ -785,24 +796,27 @@ public static void verifyToDirIsASubDirOfFromDirectory(String fromKeyName, * Check whether dst parent dir exists or not. If the parent exists, then the * source can be renamed to dst path. * - * @param volumeName volume name - * @param bucketName bucket name - * @param toKeyName destination path - * @param metaMgr metadata manager + * @param volumeName volume name + * @param bucketName bucket name + * @param toKeyName destination path + * @param ozoneManager + * @param metaMgr metadata manager * @return omDirectoryInfo object of destination path's parent * or null if parent is bucket * @throws IOException if the destination parent is not a directory. */ - public static OmKeyInfo getKeyParentDir(String volumeName, String bucketName, - String toKeyName, OMMetadataManager metaMgr) throws IOException { + public static OmKeyInfo getKeyParentDir( + String volumeName, String bucketName, String toKeyName, + OzoneManager ozoneManager, OMMetadataManager metaMgr) throws IOException { int totalDirsCount = OzoneFSUtils.getFileCount(toKeyName); // skip parent is root '/' if (totalDirsCount <= 1) { return null; } String toKeyParentDir = OzoneFSUtils.getParentDir(toKeyName); - OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists(metaMgr, - volumeName, bucketName, toKeyParentDir, 0); + OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists( + metaMgr, volumeName, bucketName, toKeyParentDir, 0, + ozoneManager.getDefaultReplicationConfig()); // check if the immediate parent exists if (toKeyParentDirStatus == null) { throw new OMException(String.format( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java index 892e17b7366a..9a2ddeb72624 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java @@ -108,9 +108,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // Validate bucket and volume exists or not. validateBucketAndVolume(omMetadataManager, volumeName, bucketName); - OzoneFileStatus keyStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName, - bucketName, keyName, 0); + OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( + omMetadataManager, volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); if (keyStatus == null) { throw new OMException("Key not found. Key:" + keyName, KEY_NOT_FOUND); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java index 5fd9aa4650c6..1fbfdfa34103 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java @@ -135,9 +135,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, validateBucketAndVolume(omMetadataManager, volumeName, bucketName); // Check if fromKey exists - OzoneFileStatus fromKeyFileStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName, - bucketName, fromKeyName, 0); + OzoneFileStatus fromKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists( + omMetadataManager, volumeName, bucketName, fromKeyName, 0, + ozoneManager.getDefaultReplicationConfig()); // case-1) fromKeyName should exist, otw throws exception if (fromKeyFileStatus == null) { @@ -159,9 +159,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OMFileRequest.verifyToDirIsASubDirOfFromDirectory(fromKeyName, toKeyName, fromKeyFileStatus.isDirectory()); - OzoneFileStatus toKeyFileStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, - volumeName, bucketName, toKeyName, 0); + OzoneFileStatus toKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists( + omMetadataManager, volumeName, bucketName, toKeyName, 0, + ozoneManager.getDefaultReplicationConfig()); // Check if toKey exists. if (toKeyFileStatus != null) { @@ -186,8 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String newToKeyName = OzoneFSUtils.appendFileNameToKeyPath(toKeyName, fromFileName); OzoneFileStatus newToOzoneFileStatus = - OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, - volumeName, bucketName, newToKeyName, 0); + OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, + volumeName, bucketName, newToKeyName, 0, + ozoneManager.getDefaultReplicationConfig()); if (newToOzoneFileStatus != null) { // case-5) If new destin '/dst/source' exists then throws exception @@ -213,7 +214,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // doesn't exist then throw exception, otw the source can be renamed to // destination path. OmKeyInfo toKeyParent = OMFileRequest.getKeyParentDir(volumeName, - bucketName, toKeyName, omMetadataManager); + bucketName, toKeyName, ozoneManager, omMetadataManager); omClientResponse = renameKey(toKeyParent, toKeyName, fromKeyValue, fromKeyName, isRenameDirectory, keyArgs.getModificationTime(), @@ -297,7 +298,7 @@ private OMClientResponse renameKey(OmKeyInfo toKeyParent, String toKeyName, setModificationTime(ommm, omBucketInfo, toKeyParent, volumeId, bucketId, modificationTime, dirTable, trxnLogIndex); fromKeyParent = OMFileRequest.getKeyParentDir(fromKeyValue.getVolumeName(), - fromKeyValue.getBucketName(), fromKeyName, metadataMgr); + fromKeyValue.getBucketName(), fromKeyName, ozoneManager, metadataMgr); if (fromKeyParent == null && omBucketInfo == null) { // Get omBucketInfo only when needed to reduce unnecessary DB IO omBucketInfo = metadataMgr.getBucketTable().get(bucketKey); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java index fa9f7a20e34e..6fb29cb4bc1b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java @@ -89,8 +89,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } lockAcquired = omMetadataManager.getLock() .acquireWriteLock(BUCKET_LOCK, volume, bucket); - OzoneFileStatus keyStatus = OMFileRequest - .getOMKeyInfoIfExists(omMetadataManager, volume, bucket, key, 0); + OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( + omMetadataManager, volume, bucket, key, 0, + ozoneManager.getDefaultReplicationConfig()); if (keyStatus == null) { throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 7636170b6cfb..8911c68bc7ee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -137,8 +137,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String keyName = deleteKeyArgs.getKeys(indexFailed); String objectKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); - OmKeyInfo omKeyInfo = - getOmKeyInfo(omMetadataManager, volumeName, bucketName, keyName); + OmKeyInfo omKeyInfo = getOmKeyInfo(ozoneManager, omMetadataManager, + volumeName, bucketName, keyName); if (omKeyInfo == null) { deleteStatus = false; @@ -154,9 +154,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, checkKeyAcls(ozoneManager, volumeName, bucketName, keyName, IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY, volumeOwner); + OzoneFileStatus fileStatus = getOzoneKeyStatus( + ozoneManager, omMetadataManager, volumeName, bucketName, keyName); addKeyToAppropriateList(omKeyInfoList, omKeyInfo, dirList, - getOzoneKeyStatus(omMetadataManager, volumeName, bucketName, - keyName)); + fileStatus); } catch (Exception ex) { deleteStatus = false; LOG.error("Acl check failed for Key: {}", objectKey, ex); @@ -241,8 +242,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } protected OzoneFileStatus getOzoneKeyStatus( - OMMetadataManager omMetadataManager, String volumeName, String bucketName, - String keyName) throws IOException { + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, + String volumeName, String bucketName, String keyName) throws IOException { // implemented in child class return null; } @@ -287,12 +288,11 @@ protected void addKeyToAppropriateList(List omKeyInfoList, omKeyInfoList.add(omKeyInfo); } - protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager, + protected OmKeyInfo getOmKeyInfo( + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, String volume, String bucket, String key) throws IOException { String objectKey = omMetadataManager.getOzoneKey(volume, bucket, key); - OmKeyInfo omKeyInfo = - omMetadataManager.getKeyTable(getBucketLayout()).get(objectKey); - return omKeyInfo; + return omMetadataManager.getKeyTable(getBucketLayout()).get(objectKey); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index 11c2ef7e89f0..7fe527bd95ea 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -62,11 +62,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } @Override - protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager, + protected OmKeyInfo getOmKeyInfo( + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, String volumeName, String bucketName, String keyName) throws IOException { - OzoneFileStatus keyStatus = - getOzoneKeyStatus(omMetadataManager, volumeName, bucketName, keyName); + OzoneFileStatus keyStatus = getOzoneKeyStatus( + ozoneManager, omMetadataManager, volumeName, bucketName, keyName); return keyStatus != null ? keyStatus.getKeyInfo() : null; } @@ -82,12 +83,11 @@ protected void addKeyToAppropriateList(List omKeyInfoList, @Override protected OzoneFileStatus getOzoneKeyStatus( - OMMetadataManager omMetadataManager, String volumeName, String bucketName, - String keyName) throws IOException { - OzoneFileStatus keyStatus = OMFileRequest - .getOMKeyInfoIfExists(omMetadataManager, volumeName, bucketName, - keyName, 0); - return keyStatus; + OzoneManager ozoneManager, OMMetadataManager omMetadataManager, + String volumeName, String bucketName, String keyName) throws IOException { + return OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, + volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java index e83486bb9072..ebd4c5dd8db4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAclRequestWithFSO.java @@ -89,8 +89,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } lockAcquired = omMetadataManager.getLock() .acquireWriteLock(BUCKET_LOCK, volume, bucket); - OzoneFileStatus keyStatus = OMFileRequest - .getOMKeyInfoIfExists(omMetadataManager, volume, bucket, key, 0); + OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( + omMetadataManager, volume, bucket, key, 0, + ozoneManager.getDefaultReplicationConfig()); if (keyStatus == null) { throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java index 93d48cb9e9fa..1ad5a0931df3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMSetTimesRequestWithFSO.java @@ -54,7 +54,8 @@ public void testDirSetTimesRequest() throws Exception { long mtime = 2000; executeAndReturn(mtime); OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists( - omMetadataManager, volumeName, bucketName, keyName, 0); + omMetadataManager, volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); assertNotNull(keyStatus); assertTrue(keyStatus.isDirectory()); long keyMtime = keyStatus.getKeyInfo().getModificationTime(); @@ -63,7 +64,8 @@ public void testDirSetTimesRequest() throws Exception { long newMtime = -1; executeAndReturn(newMtime); keyStatus = OMFileRequest.getOMKeyInfoIfExists( - omMetadataManager, volumeName, bucketName, keyName, 0); + omMetadataManager, volumeName, bucketName, keyName, 0, + ozoneManager.getDefaultReplicationConfig()); assertNotNull(keyStatus); assertTrue(keyStatus.isDirectory()); keyMtime = keyStatus.getKeyInfo().getModificationTime();