HDDS-5385. [FSO] Remove ozone.om.metadata.layout config in OM

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -2785,18 +2785,6 @@
     </description>
   </property>
 
-  <property>
-    <name>ozone.om.metadata.layout</name>
-    <tag>OZONE, OM</tag>
-    <value>SIMPLE</value>
-    <description>
-      This property is used to define the metadata layout of file system
-      paths. If it is configured as PREFIX in combination with
-      ozone.om.enable.filesystem.paths to true then this allows to perform
-      atomic rename and delete of any directory at any level in the namespace.
-      Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX.
-    </description>
-  </property>
   <property>
     <name>ozone.directory.deleting.service.interval</name>
     <value>1m</value>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -263,12 +263,6 @@ private OMConfigKeys() {
 //  atomic rename and delete of any directory at any level in the namespace.
 //  Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX.
 
-  public static final String OZONE_OM_METADATA_LAYOUT =
-          "ozone.om.metadata.layout";
-  public static final String OZONE_OM_METADATA_LAYOUT_DEFAULT = "SIMPLE";
-
-  public static final String OZONE_OM_METADATA_LAYOUT_PREFIX = "PREFIX";
-
   // Default bucket layout used by Ozone Manager during bucket creation
   // when a client does not specify the bucket layout option.
   public static final String OZONE_DEFAULT_BUCKET_LAYOUT =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java

@@ -89,8 +89,6 @@ public static void createCluster() throws IOException {
     if (fsOptimizedServer){
       conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
           true);
-      conf.set(OMConfigKeys.OZONE_OM_METADATA_LAYOUT,
-          OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
     }
     conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
         BucketLayout.LEGACY.name());

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Utility class to check standard output.
+ */
+public class TestStandardOutputUtil {
+  private final ByteArrayOutputStream outContent =
+      new ByteArrayOutputStream();
+  private final ByteArrayOutputStream errContent =
+      new ByteArrayOutputStream();
+  private final PrintStream originalOut = System.out;
+  private final PrintStream originalErr = System.err;
+  private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+  /**
+   * Set up fresh output and error streams before test.
+   *
+   * @throws UnsupportedEncodingException
+   */
+  @Before
+  public void setUpStreams() throws UnsupportedEncodingException {
+    System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+    System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+  }
+
+  /**
+   * Restore original error and output streams after test.
+   */
+  @After
+  public void restoreStreams() {
+    System.setOut(originalOut);
+    System.setErr(originalErr);
+  }
+
+  public String getOutContentString()
+      throws UnsupportedEncodingException {
+    return getOutContentString(DEFAULT_ENCODING);
+  }
+
+  public String getErrContentString()
+      throws UnsupportedEncodingException {
+    return getErrContentString(DEFAULT_ENCODING);
+  }
+
+  public String getOutContentString(String encoding)
+      throws UnsupportedEncodingException {
+    return outContent.toString(encoding);
+  }
+
+  public String getErrContentString(String encoding)
+      throws UnsupportedEncodingException {
+    return errContent.toString(encoding);
+  }
+
+  public String getDefaultEncoding() {
+    return DEFAULT_ENCODING;
+  }
+
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java

@@ -34,7 +34,6 @@
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -108,8 +107,7 @@ public class TestOzoneClientMultipartUploadWithFSO {
   @BeforeClass
   public static void init() throws Exception {
     OzoneConfiguration conf = new OzoneConfiguration();
-    TestOMRequestUtils.configureFSOptimizedPaths(conf,
-            true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+    TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
     startCluster(conf);
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java

@@ -51,6 +51,7 @@
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
@@ -94,17 +95,18 @@ public class TestReadRetries {
       storageContainerLocationClient;
 
   private static final String SCM_ID = UUID.randomUUID().toString();
-  private String layoutVersion;
+  private String bucketLayout;
 
-  public TestReadRetries(String layoutVersion) {
-    this.layoutVersion = layoutVersion;
+  public TestReadRetries(String bucketLayout) {
+    this.bucketLayout = bucketLayout;
   }
 
   @Parameterized.Parameters
   public static Collection<Object[]> data() {
     return Arrays.asList(
-            new Object[]{OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT },
-            new Object[]{OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX });
+        new Object[]{OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT},
+        new Object[]{OMConfigKeys.
+              OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED});
   }
 
   /**
@@ -116,7 +118,7 @@ public void init() throws Exception {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1);
     TestOMRequestUtils.configureFSOptimizedPaths(conf,
-            true, layoutVersion);
+            true, BucketLayout.fromString(bucketLayout));
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
         .setScmId(SCM_ID)

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java

@@ -17,7 +17,6 @@
 package org.apache.hadoop.ozone.freon;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
 
 /**
@@ -28,8 +27,7 @@ public class TestHadoopDirTreeGeneratorWithFSO
 
   protected OzoneConfiguration getOzoneConfiguration() {
     OzoneConfiguration conf = new OzoneConfiguration();
-    TestOMRequestUtils.configureFSOptimizedPaths(conf,
-            true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+    TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
     return conf;
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java

@@ -255,8 +255,7 @@ private void startCluster() throws Exception {
     // Note: OM doesn't support live config reloading
     conf.setBoolean(OZONE_ACL_ENABLED, true);
 
-    TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
-        OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+    TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
 
     cluster =
         MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId)

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java

@@ -21,34 +21,53 @@
 import org.apache.hadoop.hdds.cli.OzoneAdmin;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.TestStandardOutputUtil;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.Assert;
+
+import java.io.UnsupportedEncodingException;
+import java.util.UUID;
 
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY;
 
 /**
  * Test for Namespace CLI.
  */
-public class TestNSSummaryAdmin {
+public class TestNSSummaryAdmin extends TestStandardOutputUtil {
+  private static ObjectStore store;
 
   private static OzoneAdmin ozoneAdmin;
   private static OzoneConfiguration conf;
   private static MiniOzoneCluster cluster;
 
+  private static String volumeName;
+  private static String bucketOBS;
+  private static String bucketFSO;
+
   @BeforeClass
   public static void init() throws Exception {
     conf = new OzoneConfiguration();
-    TestOMRequestUtils.configureFSOptimizedPaths(conf, true,
-        OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX);
+    TestOMRequestUtils.configureFSOptimizedPaths(conf, true);
     conf.set(OZONE_RECON_ADDRESS_KEY, "localhost:9888");
     cluster = MiniOzoneCluster.newBuilder(conf)
         .withoutDatanodes().includeRecon(true).build();
     cluster.waitForClusterToBeReady();
+    store = cluster.getClient().getObjectStore();
+
     // Client uses server conf for this test
     ozoneAdmin = new OzoneAdmin(conf);
+
+    volumeName = UUID.randomUUID().toString();
+    bucketOBS = UUID.randomUUID().toString();
+    bucketFSO = UUID.randomUUID().toString();
+    createVolumeAndBuckets();
   }
 
   @AfterClass
@@ -58,13 +77,94 @@ public static void shutdown() {
     }
   }
 
+  /**
+   * Create OBS and FSO buckets for the tests.
+   * @throws Exception
+   */
+  private static void createVolumeAndBuckets()
+      throws Exception {
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+
+    // Create OBS bucket.
+    BucketArgs bucketArgsOBS = BucketArgs.newBuilder()
+        .setBucketLayout(BucketLayout.OBJECT_STORE)
+        .build();
+    volume.createBucket(bucketOBS, bucketArgsOBS);
+
+    // Create FSO bucket.
+    BucketArgs bucketArgsFSO = BucketArgs.newBuilder()
+        .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+        .build();
+    volume.createBucket(bucketFSO, bucketArgsFSO);
+  }
+
+  /**
+   * Test NSSummaryCLI on root path.
+   */
   @Test(timeout = 60000)
-  public void testNSSummaryCLI() {
-    String[] summaryArgs = {"namespace", "summary", "/"};
-    String[] duArgs = {"namespace", "du", "/"};
-    String[] duArgsWithOps = {"namespace", "du", "-rfn", "--length=100", "/"};
-    String[] quotaArgs = {"namespace", "quota", "/"};
-    String[] distArgs = {"namespace", "dist", "/"};
+  public void testNSSummaryCLIRoot() throws UnsupportedEncodingException {
+    // Running on root path.
+    String path = "/";
+    executeAdminCommands(path);
+    // Should throw warning - only buckets can have bucket layout.
+    Assert.assertTrue(
+        getOutContentString().contains(
+            "[Warning] Namespace CLI is only designed for FSO mode."));
+    Assert.assertTrue(getOutContentString()
+        .contains("Put more files into it to visualize DU"));
+    Assert.assertTrue(getOutContentString().contains(
+        "Put more files into it to visualize file size distribution"));
+  }
+
+  /**
+   * Test NSSummaryCLI on FILE_SYSTEM_OPTIMIZED bucket.
+   */
+  @Test(timeout = 60000)
+  public void testNSSummaryCLIFSO() throws UnsupportedEncodingException {
+    // Running on FSO Bucket.
+    String path = "/" + volumeName + "/" + bucketFSO;
+    executeAdminCommands(path);
+    // Should not throw warning, since bucket is in FSO bucket layout.
+    Assert.assertFalse(
+        getOutContentString().contains(
+            "[Warning] Namespace CLI is only designed for FSO mode."));
+    Assert.assertTrue(getOutContentString()
+        .contains("Put more files into it to visualize DU"));
+    Assert.assertTrue(getOutContentString().contains(
+        "Put more files into it to visualize file size distribution"));
+  }
+
+  /**
+   * Test NSSummaryCLI on OBJECT_STORE bucket.
+   */
+  @Test(timeout = 60000)
+  public void testNSSummaryCLIOBS() throws UnsupportedEncodingException {
+    // Running on OBS Bucket.
+    String path = "/" + volumeName + "/" + bucketOBS;
+    executeAdminCommands(path);
+    // Should throw warning, since bucket is in OBS bucket layout.
+    Assert.assertTrue(
+        getOutContentString().contains(
+            "[Warning] Namespace CLI is only designed for FSO mode."));
+    Assert.assertTrue(getOutContentString()
+        .contains("Put more files into it to visualize DU"));
+    Assert.assertTrue(getOutContentString().contains(
+        "Put more files into it to visualize file size distribution"));
+  }
+
+  /**
+   * Execute ozoneAdmin commands on given path.
+   *
+   * @param path
+   */
+  private void executeAdminCommands(String path) {
+    String[] summaryArgs = {"namespace", "summary", path};
+    String[] duArgs = {"namespace", "du", path};
+    String[] duArgsWithOps =
+        {"namespace", "du", "-rfn", "--length=100", path};
+    String[] quotaArgs = {"namespace", "quota", path};
+    String[] distArgs = {"namespace", "dist", path};
 
     ozoneAdmin.execute(summaryArgs);
     ozoneAdmin.execute(duArgs);