diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 6a97c8501d57..0a7416e69b2c 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2785,18 +2785,6 @@ - - ozone.om.metadata.layout - OZONE, OM - SIMPLE - - This property is used to define the metadata layout of file system - paths. If it is configured as PREFIX in combination with - ozone.om.enable.filesystem.paths to true then this allows to perform - atomic rename and delete of any directory at any level in the namespace. - Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX. - - ozone.directory.deleting.service.interval 1m diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index c42d64006e7f..2dd51134a97c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -263,12 +263,6 @@ private OMConfigKeys() { // atomic rename and delete of any directory at any level in the namespace. // Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX. - public static final String OZONE_OM_METADATA_LAYOUT = - "ozone.om.metadata.layout"; - public static final String OZONE_OM_METADATA_LAYOUT_DEFAULT = "SIMPLE"; - - public static final String OZONE_OM_METADATA_LAYOUT_PREFIX = "PREFIX"; - // Default bucket layout used by Ozone Manager during bucket creation // when a client does not specify the bucket layout option. public static final String OZONE_DEFAULT_BUCKET_LAYOUT = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java index a7318a0d2905..56326b45273d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java @@ -89,8 +89,6 @@ public static void createCluster() throws IOException { if (fsOptimizedServer){ conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); - conf.set(OMConfigKeys.OZONE_OM_METADATA_LAYOUT, - OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); } conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.LEGACY.name()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java new file mode 100644 index 000000000000..ec80a498da87 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + +import org.junit.After; +import org.junit.Before; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; + +/** + * Utility class to check standard output. + */ +public class TestStandardOutputUtil { + private final ByteArrayOutputStream outContent = + new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = + new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); + + /** + * Set up fresh output and error streams before test. + * + * @throws UnsupportedEncodingException + */ + @Before + public void setUpStreams() throws UnsupportedEncodingException { + System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING)); + } + + /** + * Restore original error and output streams after test. + */ + @After + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); + } + + public String getOutContentString() + throws UnsupportedEncodingException { + return getOutContentString(DEFAULT_ENCODING); + } + + public String getErrContentString() + throws UnsupportedEncodingException { + return getErrContentString(DEFAULT_ENCODING); + } + + public String getOutContentString(String encoding) + throws UnsupportedEncodingException { + return outContent.toString(encoding); + } + + public String getErrContentString(String encoding) + throws UnsupportedEncodingException { + return errContent.toString(encoding); + } + + public String getDefaultEncoding() { + return DEFAULT_ENCODING; + } + +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index c24ac6b1a429..d772a3f20f84 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -34,7 +34,6 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -108,8 +107,7 @@ public class TestOzoneClientMultipartUploadWithFSO { @BeforeClass public static void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - TestOMRequestUtils.configureFSOptimizedPaths(conf, - true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); + TestOMRequestUtils.configureFSOptimizedPaths(conf, true); startCluster(conf); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java index f94e47e73363..2e369b4d5912 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; @@ -94,17 +95,18 @@ public class TestReadRetries { storageContainerLocationClient; private static final String SCM_ID = UUID.randomUUID().toString(); - private String layoutVersion; + private String bucketLayout; - public TestReadRetries(String layoutVersion) { - this.layoutVersion = layoutVersion; + public TestReadRetries(String bucketLayout) { + this.bucketLayout = bucketLayout; } @Parameterized.Parameters public static Collection data() { return Arrays.asList( - new Object[]{OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT }, - new Object[]{OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX }); + new Object[]{OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT}, + new Object[]{OMConfigKeys. + OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED}); } /** @@ -116,7 +118,7 @@ public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); TestOMRequestUtils.configureFSOptimizedPaths(conf, - true, layoutVersion); + true, BucketLayout.fromString(bucketLayout)); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .setScmId(SCM_ID) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java index c776cefb914f..497cdc1eb4f0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.freon; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; /** @@ -28,8 +27,7 @@ public class TestHadoopDirTreeGeneratorWithFSO protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration conf = new OzoneConfiguration(); - TestOMRequestUtils.configureFSOptimizedPaths(conf, - true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); + TestOMRequestUtils.configureFSOptimizedPaths(conf, true); return conf; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java index bcbea561bc40..b432826660c6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestRecursiveAclWithFSO.java @@ -255,8 +255,7 @@ private void startCluster() throws Exception { // Note: OM doesn't support live config reloading conf.setBoolean(OZONE_ACL_ENABLED, true); - TestOMRequestUtils.configureFSOptimizedPaths(conf, true, - OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); + TestOMRequestUtils.configureFSOptimizedPaths(conf, true); cluster = MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java index 3b865f671d43..c84f6b98bec5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java @@ -21,34 +21,53 @@ import org.apache.hadoop.hdds.cli.OzoneAdmin; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.TestStandardOutputUtil; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.Assert; + +import java.io.UnsupportedEncodingException; +import java.util.UUID; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; /** * Test for Namespace CLI. */ -public class TestNSSummaryAdmin { +public class TestNSSummaryAdmin extends TestStandardOutputUtil { + private static ObjectStore store; private static OzoneAdmin ozoneAdmin; private static OzoneConfiguration conf; private static MiniOzoneCluster cluster; + private static String volumeName; + private static String bucketOBS; + private static String bucketFSO; + @BeforeClass public static void init() throws Exception { conf = new OzoneConfiguration(); - TestOMRequestUtils.configureFSOptimizedPaths(conf, true, - OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); + TestOMRequestUtils.configureFSOptimizedPaths(conf, true); conf.set(OZONE_RECON_ADDRESS_KEY, "localhost:9888"); cluster = MiniOzoneCluster.newBuilder(conf) .withoutDatanodes().includeRecon(true).build(); cluster.waitForClusterToBeReady(); + store = cluster.getClient().getObjectStore(); + // Client uses server conf for this test ozoneAdmin = new OzoneAdmin(conf); + + volumeName = UUID.randomUUID().toString(); + bucketOBS = UUID.randomUUID().toString(); + bucketFSO = UUID.randomUUID().toString(); + createVolumeAndBuckets(); } @AfterClass @@ -58,13 +77,94 @@ public static void shutdown() { } } + /** + * Create OBS and FSO buckets for the tests. + * @throws Exception + */ + private static void createVolumeAndBuckets() + throws Exception { + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + + // Create OBS bucket. + BucketArgs bucketArgsOBS = BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.OBJECT_STORE) + .build(); + volume.createBucket(bucketOBS, bucketArgsOBS); + + // Create FSO bucket. + BucketArgs bucketArgsFSO = BucketArgs.newBuilder() + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .build(); + volume.createBucket(bucketFSO, bucketArgsFSO); + } + + /** + * Test NSSummaryCLI on root path. + */ @Test(timeout = 60000) - public void testNSSummaryCLI() { - String[] summaryArgs = {"namespace", "summary", "/"}; - String[] duArgs = {"namespace", "du", "/"}; - String[] duArgsWithOps = {"namespace", "du", "-rfn", "--length=100", "/"}; - String[] quotaArgs = {"namespace", "quota", "/"}; - String[] distArgs = {"namespace", "dist", "/"}; + public void testNSSummaryCLIRoot() throws UnsupportedEncodingException { + // Running on root path. + String path = "/"; + executeAdminCommands(path); + // Should throw warning - only buckets can have bucket layout. + Assert.assertTrue( + getOutContentString().contains( + "[Warning] Namespace CLI is only designed for FSO mode.")); + Assert.assertTrue(getOutContentString() + .contains("Put more files into it to visualize DU")); + Assert.assertTrue(getOutContentString().contains( + "Put more files into it to visualize file size distribution")); + } + + /** + * Test NSSummaryCLI on FILE_SYSTEM_OPTIMIZED bucket. + */ + @Test(timeout = 60000) + public void testNSSummaryCLIFSO() throws UnsupportedEncodingException { + // Running on FSO Bucket. + String path = "/" + volumeName + "/" + bucketFSO; + executeAdminCommands(path); + // Should not throw warning, since bucket is in FSO bucket layout. + Assert.assertFalse( + getOutContentString().contains( + "[Warning] Namespace CLI is only designed for FSO mode.")); + Assert.assertTrue(getOutContentString() + .contains("Put more files into it to visualize DU")); + Assert.assertTrue(getOutContentString().contains( + "Put more files into it to visualize file size distribution")); + } + + /** + * Test NSSummaryCLI on OBJECT_STORE bucket. + */ + @Test(timeout = 60000) + public void testNSSummaryCLIOBS() throws UnsupportedEncodingException { + // Running on OBS Bucket. + String path = "/" + volumeName + "/" + bucketOBS; + executeAdminCommands(path); + // Should throw warning, since bucket is in OBS bucket layout. + Assert.assertTrue( + getOutContentString().contains( + "[Warning] Namespace CLI is only designed for FSO mode.")); + Assert.assertTrue(getOutContentString() + .contains("Put more files into it to visualize DU")); + Assert.assertTrue(getOutContentString().contains( + "Put more files into it to visualize file size distribution")); + } + + /** + * Execute ozoneAdmin commands on given path. + * + * @param path + */ + private void executeAdminCommands(String path) { + String[] summaryArgs = {"namespace", "summary", path}; + String[] duArgs = {"namespace", "du", path}; + String[] duArgsWithOps = + {"namespace", "du", "-rfn", "--length=100", path}; + String[] quotaArgs = {"namespace", "quota", path}; + String[] distArgs = {"namespace", "dist", path}; ozoneAdmin.execute(summaryArgs); ozoneAdmin.execute(duArgs); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 21077548ab09..8077d07a1a91 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -229,9 +229,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME; @@ -1352,8 +1349,6 @@ public OMMetrics getMetrics() { * Start service. */ public void start() throws IOException { - initFSOLayout(); - if (omState == State.BOOTSTRAPPING) { if (isBootstrapping) { // Check that all OM configs have been updated with the new OM info. @@ -1451,7 +1446,6 @@ public void start() throws IOException { * Restarts the service. This method re-initializes the rpc server. */ public void restart() throws IOException { - initFSOLayout(); setInstanceVariablesFromConf(); LOG.info(buildRpcServerStartMessage("OzoneManager RPC server", @@ -3607,11 +3601,6 @@ public boolean getEnableFileSystemPaths() { OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); } - public String getOMMetadataLayout() { - return configuration - .getTrimmed(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT); - } - public String getOMDefaultBucketLayout() { return this.defaultBucketLayout; } @@ -3836,19 +3825,6 @@ private void updateLayoutVersionInDB(OMLayoutVersionManager lvm, String.valueOf(lvm.getMetadataLayoutVersion())); } - private void initFSOLayout() { - // TODO: Temporary workaround for OM upgrade path and will be replaced once - // upgrade HDDS-3698 story reaches consensus. Instead of cluster level - // configuration, OM needs to check this property on every bucket level. - String metaLayout = getOMMetadataLayout(); - boolean omMetadataLayoutPrefix = StringUtils.equalsIgnoreCase(metaLayout, - OZONE_OM_METADATA_LAYOUT_PREFIX); - - String status = omMetadataLayoutPrefix ? "enabled" : "disabled"; - LOG.info("Configured {}={} and {} optimized OM FS operations", - OZONE_OM_METADATA_LAYOUT, metaLayout, status); - } - private BucketLayout getBucketLayout() { return BucketLayout.DEFAULT; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 234b4229624e..b3acaaaddb3f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -139,7 +139,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); - String omLayout = ozoneManager.getOMMetadataLayout(); OmBucketInfo omBucketInfo = null; if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout() .equals(BucketLayoutProto.LEGACY)) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java index cd5421a47e15..bec1587411bb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java @@ -550,9 +550,6 @@ public static List< HddsProtos.KeyValue> getMetadataListFSO() { "value1").build()); metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key2").setValue( "value2").build()); - metadataList.add(HddsProtos.KeyValue.newBuilder().setKey( - OMConfigKeys.OZONE_OM_METADATA_LAYOUT).setValue( - OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX).build()); metadataList.add(HddsProtos.KeyValue.newBuilder().setKey( OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS).setValue( "false").build()); @@ -1066,15 +1063,18 @@ public static long addParentsToDirTable(String volumeName, String bucketName, } public static void configureFSOptimizedPaths(Configuration conf, - boolean enableFileSystemPaths, String version) { + boolean enableFileSystemPaths) { + configureFSOptimizedPaths(conf, enableFileSystemPaths, + BucketLayout.FILE_SYSTEM_OPTIMIZED); + } + + public static void configureFSOptimizedPaths(Configuration conf, + boolean enableFileSystemPaths, + BucketLayout bucketLayout) { conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, enableFileSystemPaths); - conf.set(OMConfigKeys.OZONE_OM_METADATA_LAYOUT, version); - if (StringUtils.equalsIgnoreCase( - OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX, version)) { - conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, - BucketLayout.FILE_SYSTEM_OPTIMIZED.name()); - } + conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, + bucketLayout.name()); } private static BucketLayout getDefaultBucketLayout() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java index c066c2355b69..c4b1923b11d8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java @@ -79,7 +79,6 @@ public void setup() throws Exception { auditLogger = Mockito.mock(AuditLogger.class); when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); - when(ozoneManager.getOMMetadataLayout()).thenReturn(null); } @After diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java index a460dc8ac0df..28ba8defa18e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.om.request.bucket; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; @@ -41,8 +40,6 @@ public class TestOMBucketCreateRequestWithFSO @Test public void testValidateAndUpdateCacheWithFSO() throws Exception { - when(ozoneManager.getOMMetadataLayout()).thenReturn( - OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); when(ozoneManager.getOMDefaultBucketLayout()).thenReturn( BucketLayout.FILE_SYSTEM_OPTIMIZED.name()); String volumeName = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java index ba8d835a7abc..36a4c9125394 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestWithFSO.java @@ -90,8 +90,7 @@ public void setup() throws Exception { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath()); - TestOMRequestUtils.configureFSOptimizedPaths(ozoneConfiguration, - true, OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX); + TestOMRequestUtils.configureFSOptimizedPaths(ozoneConfiguration, true); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); when(ozoneManager.getMetrics()).thenReturn(omMetrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java index 6a85ba5319c8..3a4a16d54d33 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java @@ -106,7 +106,7 @@ public Void call() throws Exception { if (duResponse.get("status").equals("PATH_NOT_FOUND")) { printPathNotFound(); } else { - if (!parent.isFSOEnabled()) { + if (!parent.isFileSystemOptimizedBucket(path)) { printFSOReminder(); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java index 4411405518a1..9f02121c8fc8 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java @@ -80,7 +80,7 @@ public Void call() throws Exception { } else if (distResponse.get("status").equals("TYPE_NOT_APPLICABLE")) { printTypeNA("File Size Distribution"); } else { - if (!parent.isFSOEnabled()) { + if (!parent.isFileSystemOptimizedBucket(path)) { printFSOReminder(); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java index b89973e0eeaa..220365883d77 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.admin.nssummary; +import org.apache.hadoop.fs.ozone.OzoneClientUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.cli.OzoneAdmin; @@ -24,9 +25,18 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.http.HttpConfig; +import org.apache.hadoop.ozone.OFSPath; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.kohsuke.MetaInfServices; import picocli.CommandLine; +import java.io.IOException; +import java.util.HashSet; + import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_DEFAULT; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTPS_ADDRESS_DEFAULT; @@ -74,10 +84,28 @@ public Class getParentType() { return OzoneAdmin.class; } - public boolean isFSOEnabled() { - OzoneConfiguration conf = parent.getOzoneConf(); - return conf.getBoolean("ozone.om.enable.filesystem.paths", false) - && conf.get("ozone.om.metadata.layout").equalsIgnoreCase("PREFIX"); + public boolean isFileSystemOptimizedBucket(String path) throws IOException { + OFSPath ofsPath = new OFSPath(path); + + OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(getOzoneConfig()); + ObjectStore objectStore = ozoneClient.getObjectStore(); + + try { + OzoneBucket bucket = objectStore.getVolume(ofsPath.getVolumeName()) + .getBucket(ofsPath.getBucketName()); + + // Resolve the bucket layout in case this is a Link Bucket. + BucketLayout resolvedBucketLayout = + OzoneClientUtils.resolveLinkBucketLayout(bucket, objectStore, + new HashSet<>()); + + return resolvedBucketLayout.isFileSystemOptimized(); + } catch (IOException e) { + System.out.println( + "Bucket layout couldn't be verified for path: " + ofsPath + + ". Exception: " + e); + return false; + } } /** diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java index dc7d59bfb7dc..9c56924af2f6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java @@ -154,10 +154,10 @@ public static void printWithUnderline(String str, boolean newLine) { public static void printFSOReminder() { printNewLines(1); - System.out.println("[Warning] FSO is NOT enabled. " + - "Namespace CLI is only designed for FSO mode.\n" + - "To enable FSO set ozone.om.enable.filesystem.paths to true " + - "and ozone.om.metadata.layout to PREFIX."); + System.out.println( + "[Warning] Namespace CLI is only designed for FSO mode.\n" + + "Bucket being accessed must be of type FILE_SYSTEM_OPTIMIZED" + + " bucket layout."); printNewLines(1); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java index 2720411987a8..88a7b2a55408 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java @@ -80,7 +80,7 @@ public Void call() throws Exception { } else if (quotaResponse.get("status").equals("TYPE_NOT_APPLICABLE")) { printTypeNA("Quota"); } else { - if (!parent.isFSOEnabled()) { + if (!parent.isFileSystemOptimizedBucket(path)) { printFSOReminder(); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java index ce4616a7de06..c0d2ed7f0cb8 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java @@ -76,7 +76,7 @@ public Void call() throws Exception { if (summaryResponse.get("status").equals("PATH_NOT_FOUND")) { printPathNotFound(); } else { - if (!parent.isFSOEnabled()) { + if (!parent.isFileSystemOptimizedBucket(path)) { printFSOReminder(); }