HDDS-6683. Refactor OM server bucket layout configuration usage

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java

@@ -107,7 +107,6 @@ public class TestOzoneShellHA {
   private static File baseDir;
   private static File testFile;
   private static String testFilePathString;
-  private static OzoneConfiguration conf = null;
   private static MiniOzoneCluster cluster = null;
   private OzoneShell ozoneShell = null;
   private OzoneAdmin ozoneAdminShell = null;
@@ -131,8 +130,11 @@ public class TestOzoneShellHA {
    */
   @BeforeClass
   public static void init() throws Exception {
-    conf = new OzoneConfiguration();
+    OzoneConfiguration conf = new OzoneConfiguration();
+    startCluster(conf);
+  }
 
+  protected static void startCluster(OzoneConfiguration conf) throws Exception {
     String path = GenericTestUtils.getTempPath(
         TestOzoneShellHA.class.getSimpleName());
     baseDir = new File(path);
@@ -261,7 +263,7 @@ private String getLeaderOMNodeId() {
   }
 
   private String getSetConfStringFromConf(String key) {
-    return String.format("--set=%s=%s", key, conf.get(key));
+    return String.format("--set=%s=%s", key, cluster.getConf().get(key));
   }
 
   private String generateSetConfString(String key, String value) {
@@ -287,7 +289,7 @@ private String[] getHASetConfStrings(int numOfArgs) {
 
     String omNodesKey = ConfUtils.addKeySuffixes(
         OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId);
-    String omNodesVal = conf.get(omNodesKey);
+    String omNodesVal = cluster.getConf().get(omNodesKey);
     res[indexOmNodes] = generateSetConfString(omNodesKey, omNodesVal);
 
     String[] omNodesArr = omNodesVal.split(",");
@@ -392,7 +394,7 @@ public void testOzoneShCmdURIs() {
     String omLeaderNodeId = getLeaderOMNodeId();
     String omLeaderNodeAddrKey = ConfUtils.addKeySuffixes(
         OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omLeaderNodeId);
-    String omLeaderNodeAddr = conf.get(omLeaderNodeAddrKey);
+    String omLeaderNodeAddr = cluster.getConf().get(omLeaderNodeAddrKey);
     String omLeaderNodeAddrWithoutPort = omLeaderNodeAddr.split(":")[0];
 
     // Test case 2: ozone sh volume create o3://om1/volume2
@@ -560,7 +562,8 @@ private OzoneConfiguration getClientConfForOzoneTrashPolicy(
   @Test
   public void testDeleteToTrashOrSkipTrash() throws Exception {
     final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
-    OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+    OzoneConfiguration clientConf =
+        getClientConfForOFS(hostPrefix, cluster.getConf());
     OzoneFsShell shell = new OzoneFsShell(clientConf);
     FileSystem fs = FileSystem.get(clientConf);
     final String strDir1 = hostPrefix + "/volumed2t/bucket1/dir1";
@@ -634,7 +637,7 @@ public void testDeleteTrashNoSkipTrash() throws Exception {
     // (default is TrashPolicyDefault)
     final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
     OzoneConfiguration clientConf =
-            getClientConfForOzoneTrashPolicy(hostPrefix, conf);
+            getClientConfForOzoneTrashPolicy(hostPrefix, cluster.getConf());
     OzoneFsShell shell = new OzoneFsShell(clientConf);
 
     int res;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * This class tests Ozone sh shell command with FSO.
+ * Inspired by TestS3Shell
+ */
+public class TestOzoneShellHAWithFSO extends TestOzoneShellHA {
+
+  /**
+   * Create a MiniOzoneCluster for testing with using distributed Ozone
+   * handler type.
+   *
+   * @throws Exception
+   */
+  @BeforeClass
+  public static void init() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+        OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
+    startCluster(conf);
+  }
+
+  /**
+   * shutdown MiniOzoneCluster.
+   */
+  @AfterClass
+  public static void shutdownCluster() {
+    shutdown();
+  }
+}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java

@@ -51,7 +51,6 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketEncryptionInfoProto;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -152,8 +151,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
     OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
         getOmRequest());
     OmBucketInfo omBucketInfo = null;
-    if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout()
-        .equals(BucketLayoutProto.LEGACY)) {
+
+    // If the user has set a value, bucketInfo.hasBucketLayout() would be true,
+    // and we would be using this value in the else condition.
+    // If the value is not specified by the user, we would set it to the server
+    // default.
+    if (!bucketInfo.hasBucketLayout()) {
       // Bucket Layout argument was not passed during bucket creation.
       String omDefaultBucketLayout = ozoneManager.getOMDefaultBucketLayout();
       BucketLayout defaultType = BucketLayout.fromString(omDefaultBucketLayout);

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java

@@ -22,6 +22,7 @@
 import java.util.UUID;
 
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.ozone.test.LambdaTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
@@ -100,7 +101,6 @@ public void testValidateAndUpdateCacheWithNoVolume() throws Exception {
     Assert.assertNull(omMetadataManager.getBucketTable().get(bucketKey));
   }
 
-
   @Test
   public void testValidateAndUpdateCacheWithBucketAlreadyExists()
       throws Exception {
@@ -124,6 +124,21 @@ public void testValidateAndUpdateCacheWithBucketAlreadyExists()
         omResponse.getStatus());
   }
 
+  @Test
+  public void testValidateAndUpdateCacheVerifyBucketLayout() throws Exception {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+    OMBucketCreateRequest omBucketCreateRequest = doPreExecute(volumeName,
+        bucketName);
+
+    doValidateAndUpdateCache(volumeName, bucketName,
+        omBucketCreateRequest.getOmRequest());
+
+    Assert.assertEquals(BucketLayout.DEFAULT,
+        omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout());
+  }
 
   private OMBucketCreateRequest doPreExecute(String volumeName,
       String bucketName) throws Exception {

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java

@@ -44,6 +44,7 @@ public void testValidateAndUpdateCacheWithFSO() throws Exception {
         BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
+    String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
 
     Assert.assertEquals(0, omMetrics.getNumFSOBucketCreates());
 
@@ -54,6 +55,8 @@ public void testValidateAndUpdateCacheWithFSO() throws Exception {
         omBucketCreateRequest.getOmRequest());
 
     Assert.assertEquals(1, omMetrics.getNumFSOBucketCreates());
+    Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
+        omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout());
   }
 
   private OMBucketCreateRequest doPreExecute(String volumeName,