HDDS-6683. Refactor OM server bucket layout configuration usage

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java

@@ -68,8 +68,6 @@ public class TestDirectoryDeletingServiceWithFSO {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestDirectoryDeletingServiceWithFSO.class);
 
-  private static boolean isBucketFSOptimized = true;
-  private static boolean enabledFileSystemPaths = true;
   private static boolean omRatisEnabled = true;
 
   private static MiniOzoneCluster cluster;
@@ -86,20 +84,14 @@ public static void init() throws Exception {
         TimeUnit.MILLISECONDS);
     conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
     conf.setBoolean(OZONE_ACL_ENABLED, true);
-    if (isBucketFSOptimized) {
-      conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
-          BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
-    } else {
-      conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
-          enabledFileSystemPaths);
-    }
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
         .build();
     cluster.waitForClusterToBeReady();
 
     // create a volume and a bucket to be used by OzoneFileSystem
-    OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
+    OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster,
+        BucketLayout.FILE_SYSTEM_OPTIMIZED);
     volumeName = bucket.getVolumeName();
     bucketName = bucket.getName();
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java

@@ -59,7 +59,9 @@ public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster,
     BucketArgs omBucketArgs;
     BucketArgs.Builder builder = BucketArgs.newBuilder();
     builder.setStorageType(StorageType.DISK);
-    builder.setBucketLayout(bucketLayout);
+    if (bucketLayout != null) {
+      builder.setBucketLayout(bucketLayout);
+    }
     omBucketArgs = builder.build();
 
     return createVolumeAndBucket(cluster, volumeName, bucketName, bucketLayout,

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java

@@ -78,10 +78,10 @@ public void testRestartWithFSOLayout() throws Exception {
           BucketLayout.FILE_SYSTEM_OPTIMIZED);
       verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED);
 
-      // 3. verify OM default behavior with LEGACY
+      // 3. verify OM default behavior with empty
       restartCluster();
       OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster,
-          BucketLayout.LEGACY);
+          null);
       verifyBucketLayout(bucket2, BucketLayout.FILE_SYSTEM_OPTIMIZED);
 
       // 4. create bucket with OBS bucket layout and verify
@@ -122,10 +122,10 @@ public void testRestartWithOBSLayout() throws Exception {
           BucketLayout.FILE_SYSTEM_OPTIMIZED);
       verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED);
 
-      // 3. verify OM default behavior with LEGACY
+      // 3. verify OM default behavior with empty
       restartCluster();
       OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster,
-          BucketLayout.LEGACY);
+          null);
       verifyBucketLayout(bucket2, BucketLayout.OBJECT_STORE);
 
       // 4. create bucket with OBS bucket layout and verify

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java

@@ -741,9 +741,9 @@ public void testCreateBucketWithBucketLayout() throws Exception {
     Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
         bucket.getBucketLayout());
 
-    // Case 4: Bucket layout: DEFAULT
+    // Case 4: Bucket layout: Empty
     sampleBucketName = UUID.randomUUID().toString();
-    builder.setBucketLayout(BucketLayout.DEFAULT);
+    builder = BucketArgs.newBuilder();
     volume.createBucket(sampleBucketName, builder.build());
     bucket = volume.getBucket(sampleBucketName);
     Assert.assertEquals(sampleBucketName, bucket.getName());
@@ -756,7 +756,7 @@ public void testCreateBucketWithBucketLayout() throws Exception {
     volume.createBucket(sampleBucketName, builder.build());
     bucket = volume.getBucket(sampleBucketName);
     Assert.assertEquals(sampleBucketName, bucket.getName());
-    Assert.assertNotEquals(BucketLayout.LEGACY, bucket.getBucketLayout());
+    Assert.assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout());
   }
 
   private void assertKeyRenamedEx(OzoneBucket bucket, String keyName)

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java

@@ -107,7 +107,6 @@ public class TestOzoneShellHA {
   private static File baseDir;
   private static File testFile;
   private static String testFilePathString;
-  private static OzoneConfiguration conf = null;
   private static MiniOzoneCluster cluster = null;
   private OzoneShell ozoneShell = null;
   private OzoneAdmin ozoneAdminShell = null;
@@ -131,8 +130,11 @@ public class TestOzoneShellHA {
    */
   @BeforeClass
   public static void init() throws Exception {
-    conf = new OzoneConfiguration();
+    OzoneConfiguration conf = new OzoneConfiguration();
+    startCluster(conf);
+  }
 
+  protected static void startCluster(OzoneConfiguration conf) throws Exception {
     String path = GenericTestUtils.getTempPath(
         TestOzoneShellHA.class.getSimpleName());
     baseDir = new File(path);
@@ -261,7 +263,7 @@ private String getLeaderOMNodeId() {
   }
 
   private String getSetConfStringFromConf(String key) {
-    return String.format("--set=%s=%s", key, conf.get(key));
+    return String.format("--set=%s=%s", key, cluster.getConf().get(key));
   }
 
   private String generateSetConfString(String key, String value) {
@@ -287,7 +289,7 @@ private String[] getHASetConfStrings(int numOfArgs) {
 
     String omNodesKey = ConfUtils.addKeySuffixes(
         OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId);
-    String omNodesVal = conf.get(omNodesKey);
+    String omNodesVal = cluster.getConf().get(omNodesKey);
     res[indexOmNodes] = generateSetConfString(omNodesKey, omNodesVal);
 
     String[] omNodesArr = omNodesVal.split(",");
@@ -392,7 +394,7 @@ public void testOzoneShCmdURIs() {
     String omLeaderNodeId = getLeaderOMNodeId();
     String omLeaderNodeAddrKey = ConfUtils.addKeySuffixes(
         OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omLeaderNodeId);
-    String omLeaderNodeAddr = conf.get(omLeaderNodeAddrKey);
+    String omLeaderNodeAddr = cluster.getConf().get(omLeaderNodeAddrKey);
     String omLeaderNodeAddrWithoutPort = omLeaderNodeAddr.split(":")[0];
 
     // Test case 2: ozone sh volume create o3://om1/volume2
@@ -560,7 +562,8 @@ private OzoneConfiguration getClientConfForOzoneTrashPolicy(
   @Test
   public void testDeleteToTrashOrSkipTrash() throws Exception {
     final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
-    OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+    OzoneConfiguration clientConf =
+        getClientConfForOFS(hostPrefix, cluster.getConf());
     OzoneFsShell shell = new OzoneFsShell(clientConf);
     FileSystem fs = FileSystem.get(clientConf);
     final String strDir1 = hostPrefix + "/volumed2t/bucket1/dir1";
@@ -634,7 +637,7 @@ public void testDeleteTrashNoSkipTrash() throws Exception {
     // (default is TrashPolicyDefault)
     final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
     OzoneConfiguration clientConf =
-            getClientConfForOzoneTrashPolicy(hostPrefix, conf);
+            getClientConfForOzoneTrashPolicy(hostPrefix, cluster.getConf());
     OzoneFsShell shell = new OzoneFsShell(clientConf);
 
     int res;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * This class tests Ozone sh shell command with FSO.
+ * Inspired by TestS3Shell
+ */
+public class TestOzoneShellHAWithFSO extends TestOzoneShellHA {
+
+  /**
+   * Create a MiniOzoneCluster for testing with using distributed Ozone
+   * handler type.
+   *
+   * @throws Exception
+   */
+  @BeforeClass
+  public static void init() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+        OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
+    startCluster(conf);
+  }
+
+  /**
+   * shutdown MiniOzoneCluster.
+   */
+  @AfterClass
+  public static void shutdownCluster() {
+    shutdown();
+  }
+}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java

@@ -51,7 +51,6 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketEncryptionInfoProto;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -152,8 +151,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
     OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
         getOmRequest());
     OmBucketInfo omBucketInfo = null;
-    if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout()
-        .equals(BucketLayoutProto.LEGACY)) {
+
+    // bucketInfo.hasBucketLayout() would be true when user sets bucket layout.
+    // Now, OM will create bucket with the user specified bucket layout.
+    // When the value is not specified by the user, OM will use the
+    // "ozone.default.bucket.layout" configured value.
+    if (!bucketInfo.hasBucketLayout()) {
       // Bucket Layout argument was not passed during bucket creation.
       String omDefaultBucketLayout = ozoneManager.getOMDefaultBucketLayout();
       BucketLayout defaultType = BucketLayout.fromString(omDefaultBucketLayout);

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java

@@ -22,6 +22,7 @@
 import java.util.UUID;
 
 import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.ozone.test.LambdaTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
@@ -100,7 +101,6 @@ public void testValidateAndUpdateCacheWithNoVolume() throws Exception {
     Assert.assertNull(omMetadataManager.getBucketTable().get(bucketKey));
   }
 
-
   @Test
   public void testValidateAndUpdateCacheWithBucketAlreadyExists()
       throws Exception {
@@ -124,6 +124,21 @@ public void testValidateAndUpdateCacheWithBucketAlreadyExists()
         omResponse.getStatus());
   }
 
+  @Test
+  public void testValidateAndUpdateCacheVerifyBucketLayout() throws Exception {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
+
+    OMBucketCreateRequest omBucketCreateRequest = doPreExecute(volumeName,
+        bucketName);
+
+    doValidateAndUpdateCache(volumeName, bucketName,
+        omBucketCreateRequest.getOmRequest());
+
+    Assert.assertEquals(BucketLayout.LEGACY,
+        omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout());
+  }
 
   private OMBucketCreateRequest doPreExecute(String volumeName,
       String bucketName) throws Exception {

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java

@@ -44,6 +44,7 @@ public void testValidateAndUpdateCacheWithFSO() throws Exception {
         BucketLayout.FILE_SYSTEM_OPTIMIZED.name());
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
+    String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName);
 
     Assert.assertEquals(0, omMetrics.getNumFSOBucketCreates());
 
@@ -54,6 +55,8 @@ public void testValidateAndUpdateCacheWithFSO() throws Exception {
         omBucketCreateRequest.getOmRequest());
 
     Assert.assertEquals(1, omMetrics.getNumFSOBucketCreates());
+    Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED,
+        omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout());
   }
 
   private OMBucketCreateRequest doPreExecute(String volumeName,