diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java index 14a73e3b9c09..ca55f6a2f485 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java @@ -68,8 +68,6 @@ public class TestDirectoryDeletingServiceWithFSO { private static final Logger LOG = LoggerFactory.getLogger(TestDirectoryDeletingServiceWithFSO.class); - private static boolean isBucketFSOptimized = true; - private static boolean enabledFileSystemPaths = true; private static boolean omRatisEnabled = true; private static MiniOzoneCluster cluster; @@ -86,20 +84,14 @@ public static void init() throws Exception { TimeUnit.MILLISECONDS); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); conf.setBoolean(OZONE_ACL_ENABLED, true); - if (isBucketFSOptimized) { - conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, - BucketLayout.FILE_SYSTEM_OPTIMIZED.name()); - } else { - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, - enabledFileSystemPaths); - } cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .build(); cluster.waitForClusterToBeReady(); // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster, + BucketLayout.FILE_SYSTEM_OPTIMIZED); volumeName = bucket.getVolumeName(); bucketName = bucket.getName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index 4b1e72456621..eb8f69aa939d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -59,7 +59,9 @@ public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster, BucketArgs omBucketArgs; BucketArgs.Builder builder = BucketArgs.newBuilder(); builder.setStorageType(StorageType.DISK); - builder.setBucketLayout(bucketLayout); + if (bucketLayout != null) { + builder.setBucketLayout(bucketLayout); + } omBucketArgs = builder.build(); return createVolumeAndBucket(cluster, volumeName, bucketName, bucketLayout, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java index e43176021a73..214d38647da0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java @@ -78,10 +78,10 @@ public void testRestartWithFSOLayout() throws Exception { BucketLayout.FILE_SYSTEM_OPTIMIZED); verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED); - // 3. verify OM default behavior with LEGACY + // 3. verify OM default behavior with empty restartCluster(); OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster, - BucketLayout.LEGACY); + null); verifyBucketLayout(bucket2, BucketLayout.FILE_SYSTEM_OPTIMIZED); // 4. create bucket with OBS bucket layout and verify @@ -122,10 +122,10 @@ public void testRestartWithOBSLayout() throws Exception { BucketLayout.FILE_SYSTEM_OPTIMIZED); verifyBucketLayout(bucket1, BucketLayout.FILE_SYSTEM_OPTIMIZED); - // 3. verify OM default behavior with LEGACY + // 3. verify OM default behavior with empty restartCluster(); OzoneBucket bucket2 = TestDataUtil.createVolumeAndBucket(cluster, - BucketLayout.LEGACY); + null); verifyBucketLayout(bucket2, BucketLayout.OBJECT_STORE); // 4. create bucket with OBS bucket layout and verify diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index af82b7cab6c2..c0a844c520e0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java @@ -741,9 +741,9 @@ public void testCreateBucketWithBucketLayout() throws Exception { Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, bucket.getBucketLayout()); - // Case 4: Bucket layout: DEFAULT + // Case 4: Bucket layout: Empty sampleBucketName = UUID.randomUUID().toString(); - builder.setBucketLayout(BucketLayout.DEFAULT); + builder = BucketArgs.newBuilder(); volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); Assert.assertEquals(sampleBucketName, bucket.getName()); @@ -756,7 +756,7 @@ public void testCreateBucketWithBucketLayout() throws Exception { volume.createBucket(sampleBucketName, builder.build()); bucket = volume.getBucket(sampleBucketName); Assert.assertEquals(sampleBucketName, bucket.getName()); - Assert.assertNotEquals(BucketLayout.LEGACY, bucket.getBucketLayout()); + Assert.assertEquals(BucketLayout.LEGACY, bucket.getBucketLayout()); } private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index d22e4cad48e3..10df7a8d5bdc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -107,7 +107,6 @@ public class TestOzoneShellHA { private static File baseDir; private static File testFile; private static String testFilePathString; - private static OzoneConfiguration conf = null; private static MiniOzoneCluster cluster = null; private OzoneShell ozoneShell = null; private OzoneAdmin ozoneAdminShell = null; @@ -131,8 +130,11 @@ public class TestOzoneShellHA { */ @BeforeClass public static void init() throws Exception { - conf = new OzoneConfiguration(); + OzoneConfiguration conf = new OzoneConfiguration(); + startCluster(conf); + } + protected static void startCluster(OzoneConfiguration conf) throws Exception { String path = GenericTestUtils.getTempPath( TestOzoneShellHA.class.getSimpleName()); baseDir = new File(path); @@ -261,7 +263,7 @@ private String getLeaderOMNodeId() { } private String getSetConfStringFromConf(String key) { - return String.format("--set=%s=%s", key, conf.get(key)); + return String.format("--set=%s=%s", key, cluster.getConf().get(key)); } private String generateSetConfString(String key, String value) { @@ -287,7 +289,7 @@ private String[] getHASetConfStrings(int numOfArgs) { String omNodesKey = ConfUtils.addKeySuffixes( OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId); - String omNodesVal = conf.get(omNodesKey); + String omNodesVal = cluster.getConf().get(omNodesKey); res[indexOmNodes] = generateSetConfString(omNodesKey, omNodesVal); String[] omNodesArr = omNodesVal.split(","); @@ -392,7 +394,7 @@ public void testOzoneShCmdURIs() { String omLeaderNodeId = getLeaderOMNodeId(); String omLeaderNodeAddrKey = ConfUtils.addKeySuffixes( OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omLeaderNodeId); - String omLeaderNodeAddr = conf.get(omLeaderNodeAddrKey); + String omLeaderNodeAddr = cluster.getConf().get(omLeaderNodeAddrKey); String omLeaderNodeAddrWithoutPort = omLeaderNodeAddr.split(":")[0]; // Test case 2: ozone sh volume create o3://om1/volume2 @@ -560,7 +562,8 @@ private OzoneConfiguration getClientConfForOzoneTrashPolicy( @Test public void testDeleteToTrashOrSkipTrash() throws Exception { final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId; - OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf); + OzoneConfiguration clientConf = + getClientConfForOFS(hostPrefix, cluster.getConf()); OzoneFsShell shell = new OzoneFsShell(clientConf); FileSystem fs = FileSystem.get(clientConf); final String strDir1 = hostPrefix + "/volumed2t/bucket1/dir1"; @@ -634,7 +637,7 @@ public void testDeleteTrashNoSkipTrash() throws Exception { // (default is TrashPolicyDefault) final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId; OzoneConfiguration clientConf = - getClientConfForOzoneTrashPolicy(hostPrefix, conf); + getClientConfForOzoneTrashPolicy(hostPrefix, cluster.getConf()); OzoneFsShell shell = new OzoneFsShell(clientConf); int res; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java new file mode 100644 index 000000000000..fef5624726c1 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.shell; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * This class tests Ozone sh shell command with FSO. + * Inspired by TestS3Shell + */ +public class TestOzoneShellHAWithFSO extends TestOzoneShellHA { + + /** + * Create a MiniOzoneCluster for testing with using distributed Ozone + * handler type. + * + * @throws Exception + */ + @BeforeClass + public static void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, + OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED); + startCluster(conf); + } + + /** + * shutdown MiniOzoneCluster. + */ + @AfterClass + public static void shutdownCluster() { + shutdown(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 8d31ca9794c9..320957ad95db 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -51,7 +51,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketEncryptionInfoProto; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateBucketResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -152,8 +151,12 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); OmBucketInfo omBucketInfo = null; - if (bucketInfo.getBucketLayout() == null || bucketInfo.getBucketLayout() - .equals(BucketLayoutProto.LEGACY)) { + + // bucketInfo.hasBucketLayout() would be true when user sets bucket layout. + // Now, OM will create bucket with the user specified bucket layout. + // When the value is not specified by the user, OM will use the + // "ozone.default.bucket.layout" configured value. + if (!bucketInfo.hasBucketLayout()) { // Bucket Layout argument was not passed during bucket creation. String omDefaultBucketLayout = ozoneManager.getOMDefaultBucketLayout(); BucketLayout defaultType = BucketLayout.fromString(omDefaultBucketLayout); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 83b34006a7bc..f57da71bd20d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -22,6 +22,7 @@ import java.util.UUID; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.ozone.test.LambdaTestUtils; import org.junit.Assert; import org.junit.Test; @@ -100,7 +101,6 @@ public void testValidateAndUpdateCacheWithNoVolume() throws Exception { Assert.assertNull(omMetadataManager.getBucketTable().get(bucketKey)); } - @Test public void testValidateAndUpdateCacheWithBucketAlreadyExists() throws Exception { @@ -124,6 +124,21 @@ public void testValidateAndUpdateCacheWithBucketAlreadyExists() omResponse.getStatus()); } + @Test + public void testValidateAndUpdateCacheVerifyBucketLayout() throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + + OMBucketCreateRequest omBucketCreateRequest = doPreExecute(volumeName, + bucketName); + + doValidateAndUpdateCache(volumeName, bucketName, + omBucketCreateRequest.getOmRequest()); + + Assert.assertEquals(BucketLayout.LEGACY, + omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout()); + } private OMBucketCreateRequest doPreExecute(String volumeName, String bucketName) throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java index 3dbddd7deaf5..44efbe9ebf21 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestWithFSO.java @@ -44,6 +44,7 @@ public void testValidateAndUpdateCacheWithFSO() throws Exception { BucketLayout.FILE_SYSTEM_OPTIMIZED.name()); String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); Assert.assertEquals(0, omMetrics.getNumFSOBucketCreates()); @@ -54,6 +55,8 @@ public void testValidateAndUpdateCacheWithFSO() throws Exception { omBucketCreateRequest.getOmRequest()); Assert.assertEquals(1, omMetrics.getNumFSOBucketCreates()); + Assert.assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, + omMetadataManager.getBucketTable().get(bucketKey).getBucketLayout()); } private OMBucketCreateRequest doPreExecute(String volumeName,