HDDS-8543. FSO directory should reflect bucket/cluster default replication

hadoop-ozone/dist/src/main/smoketest/ec/basic.robot

@@ -28,11 +28,13 @@ ${VOLUME}    vol${PREFIX}
 *** Keywords ***
 Create Key In EC Bucket
     [arguments]    ${size}
-    ${key} =    Set Variable    /${VOLUME}/ecbucket/${size}
+    ${dir} =    Set Variable    /${VOLUME}/ecbucket/dir
+    ${key} =    Set Variable    ${dir}/${size}
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}
     Key Should Match Local File    ${key}      ${file}
     Verify Key EC Replication Config    ${key}    RS    3    2    1048576
+    Verify Key EC Replication Config    ${dir}    RS    3    2    1048576
 
 Get Disk Usage of File with EC RS Replication
                                      [arguments]    ${fileLength}    ${dataChunkCount}    ${parityChunkCount}    ${ecChunkSize}
@@ -82,29 +84,45 @@ Create 3MB Key In EC Bucket
 Create 100MB Key In EC Bucket
     Create Key In EC Bucket    100mb
 
+Create Key in Default Bucket
+    ${size} =    Set Variable    1mb
+    ${dir} =     Set Variable    /${VOLUME}/default/dir
+    ${key} =     Set Variable    ${dir}/${size}
+    ${file} =    Set Variable    /tmp/${size}
+    Create Key    ${key}    ${file}
+    Key Should Match Local File    ${key}      ${file}
+    Verify Key Replica Replication Config   ${key}     RATIS    THREE
+    Verify Key Replica Replication Config   ${dir}     RATIS    THREE
+
 Create Key in Ratis Bucket
     ${size} =    Set Variable    1mb
-    ${key} =    Set Variable    /${VOLUME}/default/${size}
+    ${dir} =     Set Variable    /${VOLUME}/ratis/dir
+    ${key} =     Set Variable    ${dir}/${size}
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}
     Key Should Match Local File    ${key}      ${file}
     Verify Key Replica Replication Config   ${key}     RATIS    THREE
+    Verify Key Replica Replication Config   ${dir}     RATIS    THREE
 
 Create Ratis Key In EC Bucket
     ${size} =    Set Variable    1mb
-    ${key} =    Set Variable    /${VOLUME}/ecbucket/${size}Ratis
+    ${dir} =     Set Variable    /${VOLUME}/ecbucket/dir2
+    ${key} =     Set Variable    ${dir}/${size}Ratis
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}    --replication=THREE --type=RATIS
     Key Should Match Local File   ${key}    ${file}
     Verify Key Replica Replication Config   ${key}    RATIS   THREE
+    Verify Key EC Replication Config    ${dir}    RS    3    2    1048576
 
 Create EC Key In Ratis Bucket
     ${size} =    Set Variable    1mb
-    ${key} =    Set Variable    /${VOLUME}/ratis/${size}EC
+    ${dir} =     Set Variable    /${VOLUME}/ratis/dir2
+    ${key} =     Set Variable    ${dir}/${size}EC
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}    --replication=rs-3-2-1024k --type=EC 
     Key Should Match Local File    ${key}    ${file}
     Verify Key EC Replication Config    ${key}    RS    3    2    1048576
+    Verify Key Replica Replication Config   ${dir}     RATIS    THREE
 
 Test Invalid Replication Parameters
     ${message} =    Execute And Ignore Error    ozone sh bucket create --replication=rs-3-2-1024k --type=RATIS /${VOLUME}/foo

hadoop-ozone/dist/src/main/smoketest/ec/read.robot

@@ -23,16 +23,16 @@ ${VOLUME}    vol${PREFIX}
 
 *** Test Cases ***
 Read 1MB EC Key
-    Key Should Match Local File    /${VOLUME}/ecbucket/1mb      /tmp/1mb
+    Key Should Match Local File    /${VOLUME}/ecbucket/dir/1mb      /tmp/1mb
 
 Read 2MB EC Key
-    Key Should Match Local File    /${VOLUME}/ecbucket/2mb      /tmp/2mb
+    Key Should Match Local File    /${VOLUME}/ecbucket/dir/2mb      /tmp/2mb
 
 Read 3MB EC Key
-    Key Should Match Local File    /${VOLUME}/ecbucket/3mb      /tmp/3mb
+    Key Should Match Local File    /${VOLUME}/ecbucket/dir/3mb      /tmp/3mb
 
 Read 100MB EC Key
-    Key Should Match Local File    /${VOLUME}/ecbucket/100mb    /tmp/100mb
+    Key Should Match Local File    /${VOLUME}/ecbucket/dir/100mb    /tmp/100mb
 
 Read EC Key in Ratis Bucket
-    Key Should Match Local File    /${VOLUME}/ratis/1mbEC    /tmp/1mb
+    Key Should Match Local File    /${VOLUME}/ratis/dir2/1mbEC    /tmp/1mb

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java

@@ -16,6 +16,7 @@
  */
 package org.apache.hadoop.ozone.om;
 
+import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -44,6 +45,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Optional;
 import java.util.UUID;
 
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE;
@@ -493,10 +495,14 @@ private void checkKeyList(String keyPrefix, String startKey,
 
     Iterator<? extends OzoneKey> ozoneKeyIterator =
         fsoBucket.listKeys(keyPrefix, startKey);
+    ReplicationConfig expectedReplication =
+        Optional.ofNullable(fsoBucket.getReplicationConfig())
+            .orElse(cluster.getOzoneManager().getDefaultReplicationConfig());
 
     List <String> keyLists = new ArrayList<>();
     while (ozoneKeyIterator.hasNext()) {
       OzoneKey ozoneKey = ozoneKeyIterator.next();
+      Assert.assertEquals(expectedReplication, ozoneKey.getReplicationConfig());
       keyLists.add(ozoneKey.getName());
     }
     LinkedList outputKeysList = new LinkedList(keyLists);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -450,7 +450,7 @@ private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName,
   private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName,
                                    String keyName) throws IOException {
     OzoneFileStatus fileStatus =
-            OMFileRequest.getOMKeyInfoIfExists(metadataManager,
+            OMFileRequest.getOMKeyInfoIfExists(ozoneManager, metadataManager,
                     volumeName, bucketName, keyName, scmBlockSize);
     if (fileStatus == null) {
       return null;
@@ -1263,8 +1263,8 @@ private OzoneFileStatus getOzoneFileStatusFSO(OmKeyArgs args,
         return new OzoneFileStatus();
       }
 
-      fileStatus = OMFileRequest.getOMKeyInfoIfExists(metadataManager,
-              volumeName, bucketName, keyName, scmBlockSize);
+      fileStatus = OMFileRequest.getOMKeyInfoIfExists(ozoneManager,
+          metadataManager, volumeName, bucketName, keyName, scmBlockSize);
 
     } finally {
       metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName,
@@ -1477,7 +1477,8 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
       Preconditions.checkArgument(!recursive);
       OzoneListStatusHelper statusHelper =
           new OzoneListStatusHelper(metadataManager, scmBlockSize,
-              this::getOzoneFileStatusFSO);
+              this::getOzoneFileStatusFSO,
+              ozoneManager.getDefaultReplicationConfig());
       Collection<OzoneFileStatus> statuses =
           statusHelper.listStatusFSO(args, startKey, numEntries,
           clientAddress, allowPartialPrefixes);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java

@@ -19,6 +19,8 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -42,6 +44,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Optional;
 import java.util.TreeMap;
 import java.util.Map;
 import java.util.PriorityQueue;
@@ -81,12 +84,18 @@ public interface ClosableIterator extends Iterator<HeapEntry>, Closeable {
   private final OMMetadataManager metadataManager;
   private final long scmBlockSize;
   private final GetFileStatusHelper getStatusHelper;
-
-  OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize,
-                        GetFileStatusHelper func) {
+  private final ReplicationConfig omDefaultReplication;
+
+  OzoneListStatusHelper(
+      OMMetadataManager metadataManager,
+      long scmBlockSize,
+      GetFileStatusHelper func,
+      ReplicationConfig omDefaultReplication
+  ) {
     this.metadataManager = metadataManager;
     this.scmBlockSize = scmBlockSize;
     this.getStatusHelper = func;
+    this.omDefaultReplication = omDefaultReplication;
   }
 
   public Collection<OzoneFileStatus> listStatusFSO(OmKeyArgs args,
@@ -191,6 +200,10 @@ public Collection<OzoneFileStatus> listStatusFSO(OmKeyArgs args,
     TreeMap<String, OzoneFileStatus> map = new TreeMap<>();
 
     BucketLayout bucketLayout = omBucketInfo.getBucketLayout();
+    ReplicationConfig replication =
+        Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig())
+            .map(DefaultReplicationConfig::getReplicationConfig)
+            .orElse(omDefaultReplication);
 
     // fetch the sorted output using a min heap iterator where
     // every remove from the heap will give the smallest entry.
@@ -200,7 +213,7 @@ public Collection<OzoneFileStatus> listStatusFSO(OmKeyArgs args,
       while (map.size() < numEntries && heapIterator.hasNext()) {
         HeapEntry entry = heapIterator.next();
         OzoneFileStatus status = entry.getStatus(prefixKey,
-            scmBlockSize, volumeName, bucketName);
+            scmBlockSize, volumeName, bucketName, replication);
         map.put(entry.key, status);
       }
     }
@@ -302,8 +315,13 @@ public int hashCode() {
       return key.hashCode();
     }
 
-    public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize,
-                                     String volumeName, String bucketName) {
+    public OzoneFileStatus getStatus(
+        String prefixPath,
+        long scmBlockSize,
+        String volumeName,
+        String bucketName,
+        ReplicationConfig bucketReplication
+    ) {
       OmKeyInfo keyInfo;
       if (entryType.isDir()) {
         Preconditions.checkArgument(value instanceof OmDirectoryInfo);
@@ -312,6 +330,7 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize,
             dirInfo.getName());
         keyInfo = OMFileRequest.getOmKeyInfo(volumeName,
             bucketName, dirInfo, dirName);
+        keyInfo.setReplicationConfig(bucketReplication); // always overwrite
       } else {
         Preconditions.checkArgument(value instanceof OmKeyInfo);
         keyInfo = (OmKeyInfo) value;

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java

@@ -26,10 +26,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import com.google.common.base.Strings;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.utils.db.BatchOperation;
 import org.apache.hadoop.hdds.utils.db.Table;
@@ -39,6 +42,7 @@
 import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
@@ -629,8 +633,9 @@ public static OmKeyInfo getOmKeyInfoFromFileTable(boolean openFileTable,
    */
   @Nullable
   public static OzoneFileStatus getOMKeyInfoIfExists(
-      OMMetadataManager omMetadataMgr, String volumeName, String bucketName,
-      String keyName, long scmBlockSize) throws IOException {
+      OzoneManager ozoneManager, OMMetadataManager omMetadataMgr,
+      String volumeName, String bucketName, String keyName,
+      long scmBlockSize) throws IOException {
 
     OMFileRequest.validateBucket(omMetadataMgr, volumeName, bucketName);
 
@@ -675,6 +680,11 @@ public static OzoneFileStatus getOMKeyInfoIfExists(
     if (omDirInfo != null) {
       OmKeyInfo omKeyInfo = getOmKeyInfo(volumeName, bucketName, omDirInfo,
               keyName);
+      ReplicationConfig replicationConfig =
+          Optional.ofNullable(omBucketInfo.getDefaultReplicationConfig())
+              .map(DefaultReplicationConfig::getReplicationConfig)
+              .orElse(ozoneManager.getDefaultReplicationConfig());
+      omKeyInfo.setReplicationConfig(replicationConfig);
       return new OzoneFileStatus(omKeyInfo, scmBlockSize, true);
     }
 
@@ -785,24 +795,26 @@ public static void verifyToDirIsASubDirOfFromDirectory(String fromKeyName,
    * Check whether dst parent dir exists or not. If the parent exists, then the
    * source can be renamed to dst path.
    *
-   * @param volumeName  volume name
-   * @param bucketName  bucket name
-   * @param toKeyName   destination path
-   * @param metaMgr     metadata manager
+   * @param volumeName volume name
+   * @param bucketName bucket name
+   * @param toKeyName destination path
+   * @param ozoneManager
+   * @param metaMgr metadata manager
    * @return omDirectoryInfo object of destination path's parent
    * or null if parent is bucket
    * @throws IOException if the destination parent is not a directory.
    */
-  public static OmKeyInfo getKeyParentDir(String volumeName, String bucketName,
-      String toKeyName, OMMetadataManager metaMgr) throws IOException {
+  public static OmKeyInfo getKeyParentDir(
+      String volumeName, String bucketName, String toKeyName,
+      OzoneManager ozoneManager, OMMetadataManager metaMgr) throws IOException {
     int totalDirsCount = OzoneFSUtils.getFileCount(toKeyName);
     // skip parent is root '/'
     if (totalDirsCount <= 1) {
       return null;
     }
     String toKeyParentDir = OzoneFSUtils.getParentDir(toKeyName);
-    OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists(metaMgr,
-            volumeName, bucketName, toKeyParentDir, 0);
+    OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists(
+        ozoneManager, metaMgr, volumeName, bucketName, toKeyParentDir, 0);
     // check if the immediate parent exists
     if (toKeyParentDirStatus == null) {
       throw new OMException(String.format(

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestWithFSO.java

@@ -108,9 +108,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
       // Validate bucket and volume exists or not.
       validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
 
-      OzoneFileStatus keyStatus =
-              OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName,
-                      bucketName, keyName, 0);
+      OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists(
+          ozoneManager, omMetadataManager, volumeName, bucketName, keyName, 0);
 
       if (keyStatus == null) {
         throw new OMException("Key not found. Key:" + keyName, KEY_NOT_FOUND);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestWithFSO.java

@@ -135,9 +135,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
       validateBucketAndVolume(omMetadataManager, volumeName, bucketName);
 
       // Check if fromKey exists
-      OzoneFileStatus fromKeyFileStatus =
-              OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName,
-                      bucketName, fromKeyName, 0);
+      OzoneFileStatus fromKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists(
+          ozoneManager, omMetadataManager, volumeName, bucketName, fromKeyName,
+          0);
 
       // case-1) fromKeyName should exist, otw throws exception
       if (fromKeyFileStatus == null) {
@@ -159,9 +159,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
       OMFileRequest.verifyToDirIsASubDirOfFromDirectory(fromKeyName,
               toKeyName, fromKeyFileStatus.isDirectory());
 
-      OzoneFileStatus toKeyFileStatus =
-              OMFileRequest.getOMKeyInfoIfExists(omMetadataManager,
-                      volumeName, bucketName, toKeyName, 0);
+      OzoneFileStatus toKeyFileStatus = OMFileRequest.getOMKeyInfoIfExists(
+          ozoneManager, omMetadataManager,
+          volumeName, bucketName, toKeyName, 0);
 
       // Check if toKey exists.
       if (toKeyFileStatus != null) {
@@ -186,8 +186,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
           String newToKeyName = OzoneFSUtils.appendFileNameToKeyPath(toKeyName,
                   fromFileName);
           OzoneFileStatus newToOzoneFileStatus =
-                  OMFileRequest.getOMKeyInfoIfExists(omMetadataManager,
-                          volumeName, bucketName, newToKeyName, 0);
+              OMFileRequest.getOMKeyInfoIfExists(
+                  ozoneManager, omMetadataManager,
+                  volumeName, bucketName, newToKeyName, 0);
 
           if (newToOzoneFileStatus != null) {
             // case-5) If new destin '/dst/source' exists then throws exception
@@ -213,7 +214,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
         // doesn't exist then throw exception, otw the source can be renamed to
         // destination path.
         OmKeyInfo toKeyParent = OMFileRequest.getKeyParentDir(volumeName,
-                bucketName, toKeyName, omMetadataManager);
+                bucketName, toKeyName, ozoneManager, omMetadataManager);
 
         omClientResponse = renameKey(toKeyParent, toKeyName, fromKeyValue,
             fromKeyName, isRenameDirectory, keyArgs.getModificationTime(),
@@ -297,7 +298,7 @@ private OMClientResponse renameKey(OmKeyInfo toKeyParent, String toKeyName,
     setModificationTime(ommm, omBucketInfo, toKeyParent, volumeId, bucketId,
         modificationTime, dirTable, trxnLogIndex);
     fromKeyParent = OMFileRequest.getKeyParentDir(fromKeyValue.getVolumeName(),
-        fromKeyValue.getBucketName(), fromKeyName, metadataMgr);
+        fromKeyValue.getBucketName(), fromKeyName, ozoneManager, metadataMgr);
     if (fromKeyParent == null && omBucketInfo == null) {
       // Get omBucketInfo only when needed to reduce unnecessary DB IO
       omBucketInfo = metadataMgr.getBucketTable().get(bucketKey);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeySetTimesRequestWithFSO.java

@@ -89,8 +89,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
       }
       lockAcquired = omMetadataManager.getLock()
           .acquireWriteLock(BUCKET_LOCK, volume, bucket);
-      OzoneFileStatus keyStatus = OMFileRequest
-          .getOMKeyInfoIfExists(omMetadataManager, volume, bucket, key, 0);
+      OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists(
+          ozoneManager, omMetadataManager, volume, bucket, key, 0);
       if (keyStatus == null) {
         throw new OMException("Key not found. Key:" + key, KEY_NOT_FOUND);
       }