From 0d223a75d27fd6fdb67f5bcb3fd89de1e1284325 Mon Sep 17 00:00:00 2001 From: wangyuanben <489341394@qq.com> Date: Tue, 19 Mar 2024 18:07:26 +0800 Subject: [PATCH 1/5] HDDS-10553. Add unit test for creating keys/files with EC replication config by ofs/o3fs --- .../docs/content/feature/ErasureCoding.md | 18 ++++++++++++ .../fs/ozone/AbstractOzoneFileSystemTest.java | 29 ++++++++++++++++--- .../AbstractRootedOzoneFileSystemTest.java | 20 +++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/docs/content/feature/ErasureCoding.md b/hadoop-hdds/docs/content/feature/ErasureCoding.md index 77866762f6d3..c4d3739f1dcd 100644 --- a/hadoop-hdds/docs/content/feature/ErasureCoding.md +++ b/hadoop-hdds/docs/content/feature/ErasureCoding.md @@ -174,7 +174,9 @@ the configuration keys `ozone.server.default.replication.type` and `ozone.server ozone.server.default.replication.type EC +``` +```XML ozone.server.default.replication RS-6-3-1024k @@ -208,6 +210,22 @@ We can pass the EC Replication Config while creating the keys irrespective of bu ozone sh key put --type EC --replication rs-6-3-1024k ``` +When using ofs/o3fs, we can pass the EC Replication Config by setting the configuration keys `ozone.replication.type` and `ozone.replication`. + +```XML + + ozone.replication.type + EC + +``` + +```XML + + ozone.replication + rs-3-2-1024k + +``` + In the case bucket already has default EC Replication Config, there is no need of passing EC Replication Config while creating key. ### Enable Intel ISA-L diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index b1af31e53f8e..307c85dd02d9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -113,6 +113,8 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; @@ -165,6 +167,7 @@ abstract class AbstractOzoneFileSystemTest { private OzoneManagerProtocol writeClient; private FileSystem fs; private OzoneFileSystem o3fs; + private OzoneConfiguration conf; private OzoneFSStorageStatistics statistics; private OzoneBucket ozoneBucket; private String volumeName; @@ -174,7 +177,8 @@ abstract class AbstractOzoneFileSystemTest { @BeforeAll void init() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); + conf = new OzoneConfiguration(); + conf.set("fs.o3fs.impl.disable.cache", "true"); conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2); @@ -426,6 +430,23 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { assertTrue(fs.getFileStatus(parent).isDirectory(), "Parent directory does not appear to be a directory"); } + @Test + public void testCreateKeyWithECReplicationConfig() throws Exception { + Path root = new Path("/" + volumeName + "/" + bucketName); + Path testKeyPath = new Path(root, "testKey"); + conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); + conf.set(OZONE_REPLICATION_TYPE, "EC"); + FileSystem fileSystem = FileSystem.get(conf); + ContractTestUtils.touch(fileSystem, testKeyPath); + + OzoneKeyDetails key = getKey(testKeyPath, false); + assertEquals(HddsProtos.ReplicationType.EC, + key.getReplicationConfig().getReplicationType()); + assertEquals("rs-3-2-1024k", + key.getReplicationConfig().getReplication()); + fileSystem.close(); + } + @Test public void testDeleteCreatesFakeParentDir() throws Exception { Path grandparent = new Path("/testDeleteCreatesFakeParentDir"); @@ -1451,10 +1472,10 @@ public void testCreateKeyShouldUseRefreshedBucketReplicationConfig() bucket.getVolumeName()); // Set the fs.defaultFS and start the filesystem - Configuration conf = new OzoneConfiguration(cluster.getConf()); - conf.set(FS_DEFAULT_NAME_KEY, rootPath); + Configuration configuration = new OzoneConfiguration(cluster.getConf()); + configuration.set(FS_DEFAULT_NAME_KEY, rootPath); // Set the number of keys to be processed during batch operate. - OzoneFileSystem o3FS = (OzoneFileSystem) FileSystem.get(conf); + OzoneFileSystem o3FS = (OzoneFileSystem) FileSystem.get(configuration); //Let's reset the clock to control the time. ((BasicOzoneClientAdapterImpl) (o3FS.getAdapter())).setClock(testClock); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index b77be05f6737..1dc5ead06702 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -118,6 +118,8 @@ import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_OFS_SHARED_TMP_DIR; @@ -232,6 +234,7 @@ public Path getBucketPath() { @BeforeAll void initClusterAndEnv() throws IOException, InterruptedException, TimeoutException { conf = new OzoneConfiguration(); + conf.set("fs.ofs.impl.disable.cache", "true"); conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2); @@ -325,6 +328,23 @@ void testCreateDoesNotAddParentDirKeys() throws Exception { fs.delete(grandparent, true); } + @Test + public void testCreateKeyWithECReplicationConfig() throws Exception { + String testKeyName = "testKey"; + Path testKeyPath = new Path(bucketPath, testKeyName); + conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); + conf.set(OZONE_REPLICATION_TYPE, "EC"); + FileSystem fileSystem = FileSystem.get(conf); + ContractTestUtils.touch(fileSystem, testKeyPath); + + OzoneKeyDetails key = getKey(testKeyPath, false); + assertEquals(HddsProtos.ReplicationType.EC, + key.getReplicationConfig().getReplicationType()); + assertEquals("rs-3-2-1024k", + key.getReplicationConfig().getReplication()); + fileSystem.close(); + } + @Test void testListStatusWithIntermediateDirWithECEnabled() throws Exception { From 4a4a7d5a4a43af1633460449538779797abe67ac Mon Sep 17 00:00:00 2001 From: wangyuanben <489341394@qq.com> Date: Wed, 20 Mar 2024 16:08:29 +0800 Subject: [PATCH 2/5] take advice --- .../fs/ozone/AbstractOzoneFileSystemTest.java | 19 ++++++------------- .../AbstractRootedOzoneFileSystemTest.java | 10 ++-------- .../hadoop/fs/ozone/OzoneFileSystemTests.java | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index 307c85dd02d9..1cc7ee21a6ba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -110,11 +110,10 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; +import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; @@ -167,7 +166,6 @@ abstract class AbstractOzoneFileSystemTest { private OzoneManagerProtocol writeClient; private FileSystem fs; private OzoneFileSystem o3fs; - private OzoneConfiguration conf; private OzoneFSStorageStatistics statistics; private OzoneBucket ozoneBucket; private String volumeName; @@ -177,8 +175,7 @@ abstract class AbstractOzoneFileSystemTest { @BeforeAll void init() throws Exception { - conf = new OzoneConfiguration(); - conf.set("fs.o3fs.impl.disable.cache", "true"); + OzoneConfiguration conf = new OzoneConfiguration(); conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2); @@ -434,17 +431,13 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { public void testCreateKeyWithECReplicationConfig() throws Exception { Path root = new Path("/" + volumeName + "/" + bucketName); Path testKeyPath = new Path(root, "testKey"); - conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); - conf.set(OZONE_REPLICATION_TYPE, "EC"); - FileSystem fileSystem = FileSystem.get(conf); - ContractTestUtils.touch(fileSystem, testKeyPath); + createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath); OzoneKeyDetails key = getKey(testKeyPath, false); assertEquals(HddsProtos.ReplicationType.EC, key.getReplicationConfig().getReplicationType()); assertEquals("rs-3-2-1024k", key.getReplicationConfig().getReplication()); - fileSystem.close(); } @Test @@ -1472,10 +1465,10 @@ public void testCreateKeyShouldUseRefreshedBucketReplicationConfig() bucket.getVolumeName()); // Set the fs.defaultFS and start the filesystem - Configuration configuration = new OzoneConfiguration(cluster.getConf()); - configuration.set(FS_DEFAULT_NAME_KEY, rootPath); + Configuration conf = new OzoneConfiguration(cluster.getConf()); + conf.set(FS_DEFAULT_NAME_KEY, rootPath); // Set the number of keys to be processed during batch operate. - OzoneFileSystem o3FS = (OzoneFileSystem) FileSystem.get(configuration); + OzoneFileSystem o3FS = (OzoneFileSystem) FileSystem.get(conf); //Let's reset the clock to control the time. ((BasicOzoneClientAdapterImpl) (o3FS.getAdapter())).setClock(testClock); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 1dc5ead06702..93be15b6afb6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -115,11 +115,10 @@ import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; +import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration; import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_OFS_SHARED_TMP_DIR; @@ -234,7 +233,6 @@ public Path getBucketPath() { @BeforeAll void initClusterAndEnv() throws IOException, InterruptedException, TimeoutException { conf = new OzoneConfiguration(); - conf.set("fs.ofs.impl.disable.cache", "true"); conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL); conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2); @@ -332,17 +330,13 @@ void testCreateDoesNotAddParentDirKeys() throws Exception { public void testCreateKeyWithECReplicationConfig() throws Exception { String testKeyName = "testKey"; Path testKeyPath = new Path(bucketPath, testKeyName); - conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); - conf.set(OZONE_REPLICATION_TYPE, "EC"); - FileSystem fileSystem = FileSystem.get(conf); - ContractTestUtils.touch(fileSystem, testKeyPath); + createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath); OzoneKeyDetails key = getKey(testKeyPath, false); assertEquals(HddsProtos.ReplicationType.EC, key.getReplicationConfig().getReplicationType()); assertEquals("rs-3-2-1024k", key.getReplicationConfig().getReplication()); - fileSystem.close(); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java index d729251267ea..732e02d11b00 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import java.io.IOException; @@ -30,6 +31,8 @@ import java.util.TreeSet; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -95,4 +98,20 @@ private static void listStatusIterator(FileSystem subject, assertEquals(total, iCount); } + + static void createKeyWithECReplicationConfiguration(OzoneConfiguration conf, Path keyPath) throws IOException { + conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); + conf.set(OZONE_REPLICATION_TYPE, "EC"); + URI uri = FileSystem.getDefaultUri(conf); + conf.setBoolean( + String.format("fs.%s.impl.disable.cache", uri.getScheme()), true); + FileSystem fileSystem = FileSystem.get(uri, conf); + ContractTestUtils.touch(fileSystem, keyPath); + + // Refresh the cache of FileSystem + conf.unset(OZONE_REPLICATION); + conf.unset(OZONE_REPLICATION_TYPE); + fileSystem = FileSystem.get(uri, conf); + fileSystem.close(); + } } From c9ef5bdc922858d4e238c09935ff92cff927d36b Mon Sep 17 00:00:00 2001 From: wangyuanben <489341394@qq.com> Date: Wed, 20 Mar 2024 21:51:59 +0800 Subject: [PATCH 3/5] take advice 2 --- .../fs/ozone/AbstractOzoneFileSystemTest.java | 12 +----- .../AbstractRootedOzoneFileSystemTest.java | 12 +----- .../hadoop/fs/ozone/OzoneFileSystemTests.java | 37 ++++++++++++++----- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index 1cc7ee21a6ba..a2a341255911 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -110,7 +110,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; -import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration; +import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.testCreateKeyWithECReplicationConfiguration; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; @@ -429,15 +429,7 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { @Test public void testCreateKeyWithECReplicationConfig() throws Exception { - Path root = new Path("/" + volumeName + "/" + bucketName); - Path testKeyPath = new Path(root, "testKey"); - createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath); - - OzoneKeyDetails key = getKey(testKeyPath, false); - assertEquals(HddsProtos.ReplicationType.EC, - key.getReplicationConfig().getReplicationType()); - assertEquals("rs-3-2-1024k", - key.getReplicationConfig().getReplication()); + testCreateKeyWithECReplicationConfiguration(cluster.getConf(), volumeName, bucketName, client); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 93be15b6afb6..ddd7a9580e9f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -115,7 +115,7 @@ import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; -import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration; +import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.testCreateKeyWithECReplicationConfiguration; import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; @@ -328,15 +328,7 @@ void testCreateDoesNotAddParentDirKeys() throws Exception { @Test public void testCreateKeyWithECReplicationConfig() throws Exception { - String testKeyName = "testKey"; - Path testKeyPath = new Path(bucketPath, testKeyName); - createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath); - - OzoneKeyDetails key = getKey(testKeyPath, false); - assertEquals(HddsProtos.ReplicationType.EC, - key.getReplicationConfig().getReplicationType()); - assertEquals("rs-3-2-1024k", - key.getReplicationConfig().getReplication()); + testCreateKeyWithECReplicationConfiguration(cluster.getConf(), volumeName, bucketName, client); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java index 732e02d11b00..2adde82d3dff 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java @@ -23,6 +23,10 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; import java.io.IOException; import java.net.URI; @@ -99,19 +103,34 @@ private static void listStatusIterator(FileSystem subject, assertEquals(total, iCount); } - static void createKeyWithECReplicationConfiguration(OzoneConfiguration conf, Path keyPath) throws IOException { + static void testCreateKeyWithECReplicationConfiguration(OzoneConfiguration inputConf, + String volumeName, String bucketName, OzoneClient client) throws IOException { + String keyName = "testKey"; + Path keyPath = new Path("/" + volumeName + "/" + bucketName + "/" + keyName); + OzoneConfiguration conf = new OzoneConfiguration(inputConf); conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); conf.set(OZONE_REPLICATION_TYPE, "EC"); URI uri = FileSystem.getDefaultUri(conf); conf.setBoolean( String.format("fs.%s.impl.disable.cache", uri.getScheme()), true); - FileSystem fileSystem = FileSystem.get(uri, conf); - ContractTestUtils.touch(fileSystem, keyPath); - - // Refresh the cache of FileSystem - conf.unset(OZONE_REPLICATION); - conf.unset(OZONE_REPLICATION_TYPE); - fileSystem = FileSystem.get(uri, conf); - fileSystem.close(); + try (FileSystem fileSystem = FileSystem.get(uri, conf)) { + ContractTestUtils.touch(fileSystem, keyPath); + OzoneKeyDetails key; + if (uri.getScheme().equals(OzoneConsts.OZONE_OFS_URI_SCHEME)) { + key = client.getObjectStore() + .getVolume(volumeName) + .getBucket(bucketName) + .getKey(keyName); + } else { + key = client.getObjectStore() + .getVolume(volumeName) + .getBucket(bucketName) + .getKey(volumeName + "/" + bucketName + "/" + keyName); + } + assertEquals(HddsProtos.ReplicationType.EC, + key.getReplicationConfig().getReplicationType()); + assertEquals("rs-3-2-1024k", + key.getReplicationConfig().getReplication()); + } } } From d3f131c5f9992ba469d53d7fa0e326c8e5707a08 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 21 Mar 2024 08:44:40 +0100 Subject: [PATCH 4/5] Revert "take advice 2" This reverts commit c9ef5bdc922858d4e238c09935ff92cff927d36b. --- .../fs/ozone/AbstractOzoneFileSystemTest.java | 12 +++++- .../AbstractRootedOzoneFileSystemTest.java | 12 +++++- .../hadoop/fs/ozone/OzoneFileSystemTests.java | 37 +++++-------------- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index a2a341255911..1cc7ee21a6ba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -110,7 +110,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; -import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.testCreateKeyWithECReplicationConfiguration; +import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; @@ -429,7 +429,15 @@ public void testCreateDoesNotAddParentDirKeys() throws Exception { @Test public void testCreateKeyWithECReplicationConfig() throws Exception { - testCreateKeyWithECReplicationConfiguration(cluster.getConf(), volumeName, bucketName, client); + Path root = new Path("/" + volumeName + "/" + bucketName); + Path testKeyPath = new Path(root, "testKey"); + createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath); + + OzoneKeyDetails key = getKey(testKeyPath, false); + assertEquals(HddsProtos.ReplicationType.EC, + key.getReplicationConfig().getReplicationType()); + assertEquals("rs-3-2-1024k", + key.getReplicationConfig().getReplication()); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index ddd7a9580e9f..93be15b6afb6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -115,7 +115,7 @@ import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; -import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.testCreateKeyWithECReplicationConfiguration; +import static org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration; import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; @@ -328,7 +328,15 @@ void testCreateDoesNotAddParentDirKeys() throws Exception { @Test public void testCreateKeyWithECReplicationConfig() throws Exception { - testCreateKeyWithECReplicationConfiguration(cluster.getConf(), volumeName, bucketName, client); + String testKeyName = "testKey"; + Path testKeyPath = new Path(bucketPath, testKeyName); + createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath); + + OzoneKeyDetails key = getKey(testKeyPath, false); + assertEquals(HddsProtos.ReplicationType.EC, + key.getReplicationConfig().getReplicationType()); + assertEquals("rs-3-2-1024k", + key.getReplicationConfig().getReplication()); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java index 2adde82d3dff..732e02d11b00 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java @@ -23,10 +23,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; import java.io.IOException; import java.net.URI; @@ -103,34 +99,19 @@ private static void listStatusIterator(FileSystem subject, assertEquals(total, iCount); } - static void testCreateKeyWithECReplicationConfiguration(OzoneConfiguration inputConf, - String volumeName, String bucketName, OzoneClient client) throws IOException { - String keyName = "testKey"; - Path keyPath = new Path("/" + volumeName + "/" + bucketName + "/" + keyName); - OzoneConfiguration conf = new OzoneConfiguration(inputConf); + static void createKeyWithECReplicationConfiguration(OzoneConfiguration conf, Path keyPath) throws IOException { conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); conf.set(OZONE_REPLICATION_TYPE, "EC"); URI uri = FileSystem.getDefaultUri(conf); conf.setBoolean( String.format("fs.%s.impl.disable.cache", uri.getScheme()), true); - try (FileSystem fileSystem = FileSystem.get(uri, conf)) { - ContractTestUtils.touch(fileSystem, keyPath); - OzoneKeyDetails key; - if (uri.getScheme().equals(OzoneConsts.OZONE_OFS_URI_SCHEME)) { - key = client.getObjectStore() - .getVolume(volumeName) - .getBucket(bucketName) - .getKey(keyName); - } else { - key = client.getObjectStore() - .getVolume(volumeName) - .getBucket(bucketName) - .getKey(volumeName + "/" + bucketName + "/" + keyName); - } - assertEquals(HddsProtos.ReplicationType.EC, - key.getReplicationConfig().getReplicationType()); - assertEquals("rs-3-2-1024k", - key.getReplicationConfig().getReplication()); - } + FileSystem fileSystem = FileSystem.get(uri, conf); + ContractTestUtils.touch(fileSystem, keyPath); + + // Refresh the cache of FileSystem + conf.unset(OZONE_REPLICATION); + conf.unset(OZONE_REPLICATION_TYPE); + fileSystem = FileSystem.get(uri, conf); + fileSystem.close(); } } From 6e9ac65f8b906d9ccb908f1a72f43552537ee853 Mon Sep 17 00:00:00 2001 From: wangyuanben <489341394@qq.com> Date: Thu, 21 Mar 2024 08:53:20 +0100 Subject: [PATCH 5/5] copy input config, close filesystem --- .../hadoop/fs/ozone/OzoneFileSystemTests.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java index 732e02d11b00..47c584e048a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java @@ -99,19 +99,16 @@ private static void listStatusIterator(FileSystem subject, assertEquals(total, iCount); } - static void createKeyWithECReplicationConfiguration(OzoneConfiguration conf, Path keyPath) throws IOException { + static void createKeyWithECReplicationConfiguration(OzoneConfiguration inputConf, Path keyPath) + throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(inputConf); conf.set(OZONE_REPLICATION, "rs-3-2-1024k"); conf.set(OZONE_REPLICATION_TYPE, "EC"); URI uri = FileSystem.getDefaultUri(conf); conf.setBoolean( String.format("fs.%s.impl.disable.cache", uri.getScheme()), true); - FileSystem fileSystem = FileSystem.get(uri, conf); - ContractTestUtils.touch(fileSystem, keyPath); - - // Refresh the cache of FileSystem - conf.unset(OZONE_REPLICATION); - conf.unset(OZONE_REPLICATION_TYPE); - fileSystem = FileSystem.get(uri, conf); - fileSystem.close(); + try (FileSystem fileSystem = FileSystem.get(uri, conf)) { + ContractTestUtils.touch(fileSystem, keyPath); + } } }