From 93098bbd454f2ae5a6758a9914d7ca7c55aefb61 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sat, 5 Apr 2025 13:38:42 +0800 Subject: [PATCH 1/5] HDDS-12525. Replace TestHelper#createKey with TestDataUtil#createKey --- .../org/apache/hadoop/ozone/om/TestKeyPurging.java | 11 ++--------- .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 +++----- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 14 ++++---------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java index 37d22b1720d9..96da1ce901bd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java @@ -28,19 +28,16 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; @@ -108,11 +105,7 @@ public void testKeysPurgingByKeyDeletingService() throws Exception { for (int i = 1; i <= NUM_KEYS; i++) { String keyName = keyBase + "-" + i; keys.add(keyName); - OzoneOutputStream keyStream = TestHelper.createKey( - keyName, ReplicationType.RATIS, ReplicationFactor.ONE, - KEY_SIZE, store, volumeName, bucketName); - keyStream.write(data); - keyStream.close(); + TestDataUtil.createKey(bucket, keyName, data); } // Delete created keys diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java index cd2fc4af2949..971f277b7148 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -391,11 +392,8 @@ private void writeTestData(String volumeName, String keyString = UUID.randomUUID().toString(); byte[] data = ContainerTestHelper.getFixedLengthString( keyString, 100).getBytes(UTF_8); - OzoneOutputStream keyStream = TestHelper.createKey( - keyName, ReplicationType.RATIS, ReplicationFactor.ONE, - 100, store, volumeName, bucketName); - keyStream.write(data); - keyStream.close(); + TestDataUtil.createKey(store.getVolume(volumeName). + getBucket(bucketName), keyName, data); } private void assertKeysWritten(String volumeName, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index 04c59e6f087a..b8774cb00996 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java @@ -24,19 +24,16 @@ import java.util.UUID; import javax.ws.rs.core.Response; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.recon.api.NSSummaryEndpoint; import org.apache.hadoop.ozone.recon.api.types.EntityType; @@ -67,7 +64,7 @@ public static void init() throws Exception { OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED); cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(1) + .setNumDatanodes(3) .includeRecon(true) .build(); cluster.waitForClusterToBeReady(); @@ -92,11 +89,8 @@ private void writeTestData(String volumeName, String keyString = UUID.randomUUID().toString(); byte[] data = ContainerTestHelper.getFixedLengthString( keyString, 100).getBytes(UTF_8); - OzoneOutputStream keyStream = TestHelper.createKey( - keyName, ReplicationType.RATIS, ReplicationFactor.ONE, - 100, store, volumeName, bucketName); - keyStream.write(data); - keyStream.close(); + TestDataUtil.createKey(store.getVolume(volumeName). + getBucket(bucketName), keyName, data); } private void writeKeys(String vol, String bucket, String key) From e1f774c3d27fed7fdb56e893531a094ddd8a12d7 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sat, 5 Apr 2025 15:27:26 +0800 Subject: [PATCH 2/5] Fixed styles --- .../org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java index 971f277b7148..edc9b569b2a5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java @@ -39,16 +39,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus; From 1d2e401cb64fbb7209c24b6bb68cf657fc55db7c Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sun, 6 Apr 2025 13:24:50 +0800 Subject: [PATCH 3/5] Refactor the testHelp.createkey() --- .../hadoop/hdds/scm/TestWatchForCommit.java | 14 ++++---- .../org/apache/hadoop/ozone/TestDataUtil.java | 6 ++++ .../client/rpc/TestBlockOutputStream.java | 7 ++-- .../TestCloseContainerHandlingByClient.java | 9 +++-- .../rpc/TestDiscardPreallocatedBlocks.java | 8 +++-- .../client/rpc/TestECKeyOutputStream.java | 19 ++++++----- .../rpc/TestFailureHandlingByClient.java | 9 +++-- ...TestFailureHandlingByClientFlushDelay.java | 9 +++-- .../TestMultiBlockWritesWithDnFailures.java | 9 +++-- ...oneClientRetriesOnExceptionFlushDelay.java | 9 +++-- .../TestOzoneClientRetriesOnExceptions.java | 9 +++-- .../hadoop/ozone/container/TestHelper.java | 34 ------------------- ...stContainerScannerIntegrationAbstract.java | 9 +++-- 13 files changed, 80 insertions(+), 71 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java index 136a640d7c33..7e9e424c36f6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; +import static org.apache.hadoop.ozone.TestDataUtil.createKey; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -35,6 +36,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -180,7 +183,10 @@ private String getKeyName() { @Test public void testWatchForCommitWithKeyWrite() throws Exception { String keyName = getKeyName(); - OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0); + + OzoneOutputStream key = createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), 0); int dataLength = maxFlushSize + 50; // write data more than 1 chunk byte[] data1 = @@ -392,12 +398,6 @@ public void testWatchForCommitForGroupMismatchException() throws Exception { } } - private OzoneOutputStream createKey(String keyName, ReplicationType type, - long size) throws Exception { - return TestHelper - .createKey(keyName, type, size, objectStore, volumeName, bucketName); - } - private void validateData(String keyName, byte[] data) throws Exception { TestHelper .validateData(keyName, data, objectStore, volumeName, bucketName); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index ab6e7de9c735..a58668b20001 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.VolumeArgs; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -134,6 +135,11 @@ public static void createKey(OzoneBucket bucket, String keyName, } } + public static OzoneOutputStream createKey(OzoneBucket bucket, String keyName, + ReplicationConfig replicationConfig, long size) throws Exception { + return bucket.createKey(keyName, size, replicationConfig, new HashMap<>()); + } + public static String getKey(OzoneBucket bucket, String keyName) throws IOException { try (InputStream stream = bucket.readKey(keyName)) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index a67dca9e8eaf..bf7ec52e99f5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Stream; import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -52,6 +53,7 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -779,8 +781,9 @@ static OzoneOutputStream createKey(OzoneClient client, String keyName) static OzoneOutputStream createKey(OzoneClient client, String keyName, long size, ReplicationFactor factor) throws Exception { - return TestHelper.createKey(keyName, ReplicationType.RATIS, factor, size, - client.getObjectStore(), VOLUME, BUCKET); + ReplicationConfig config = ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, factor); + return TestDataUtil.createKey(client.getObjectStore().getVolume(VOLUME).getBucket(BUCKET), + keyName, config, size); } static String getKeyName() { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index 26f596c44335..f2ed50fae052 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -31,10 +31,13 @@ import java.util.UUID; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -321,8 +324,10 @@ private void waitForContainerClose(OzoneOutputStream outputStream) private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestHelper - .createKey(keyName, type, size, objectStore, volumeName, bucketName); + return TestDataUtil + .createKey(objectStore. + getVolume(volumeName).getBucket(bucketName), keyName, + ReplicationConfig.fromTypeAndFactor(type, ReplicationFactor.THREE), size); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java index ce78960918cc..94ad7bb134f3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java @@ -20,7 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; -import static org.apache.hadoop.ozone.container.TestHelper.createKey; +import static org.apache.hadoop.ozone.TestDataUtil.createKey; import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerClose; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -30,6 +30,8 @@ import java.util.List; import java.util.UUID; import org.apache.hadoop.conf.StorageUnit; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -89,7 +91,9 @@ void cleanup() { public void testDiscardPreallocatedBlocks() throws Exception { String keyName = "key"; OzoneOutputStream key = - createKey(keyName, ReplicationType.RATIS, 2L * blockSize, objectStore, volumeName, bucketName); + createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), 2L * blockSize); KeyOutputStream keyOutputStream = assertInstanceOf(KeyOutputStream.class, key.getOutputStream()); // With the initial size provided, it should have pre allocated 2 blocks diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java index 2bc5947a7000..9382dc86f766 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java @@ -61,6 +61,7 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -73,7 +74,6 @@ import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.ozone.test.GenericTestUtils; @@ -181,10 +181,10 @@ public static void shutdown() { @Test public void testCreateKeyWithECReplicationConfig() throws Exception { - try (OzoneOutputStream key = TestHelper - .createKey(keyString, new ECReplicationConfig(3, 2, - ECReplicationConfig.EcCodec.RS, chunkSize), inputSize, - objectStore, volumeName, bucketName)) { + try (OzoneOutputStream key = TestDataUtil + .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyString, new ECReplicationConfig(3, 2, + ECReplicationConfig.EcCodec.RS, chunkSize), inputSize)) { assertInstanceOf(ECKeyOutputStream.class, key.getOutputStream()); } } @@ -212,8 +212,9 @@ public void testECKeyCreatetWithDatanodeIdChange() ObjectStore store = client1.getObjectStore(); store.createVolume(volumeName); store.getVolume(volumeName).createBucket(bucketName); - OzoneOutputStream key = TestHelper.createKey(keyString, new ECReplicationConfig(3, 2, - ECReplicationConfig.EcCodec.RS, 1024), inputSize, store, volumeName, bucketName); + OzoneOutputStream key = TestDataUtil.createKey(store.getVolume(volumeName).getBucket(bucketName), + keyString, new ECReplicationConfig(3, 2, + ECReplicationConfig.EcCodec.RS, 1024), inputSize); byte[] b = new byte[6 * 1024]; ECKeyOutputStream groupOutputStream = (ECKeyOutputStream) key.getOutputStream(); List locationInfoList = groupOutputStream.getLocationInfoList(); @@ -573,9 +574,9 @@ private byte[] getInputBytes(int offset, int bufferChunks, int numChunks) { @Test public void testBlockedHflushAndHsync() throws Exception { // Expect ECKeyOutputStream hflush and hsync calls to throw exception - try (OzoneOutputStream oOut = TestHelper.createKey( + try (OzoneOutputStream oOut = TestDataUtil.createKey(objectStore.getVolume(volumeName).getBucket(bucketName), keyString, new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, chunkSize), - inputSize, objectStore, volumeName, bucketName)) { + inputSize)) { assertInstanceOf(ECKeyOutputStream.class, oOut.getOutputStream()); KeyOutputStream kOut = (KeyOutputStream) oOut.getOutputStream(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java index 30446af41b0e..c09584e3582d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java @@ -36,6 +36,8 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -53,6 +55,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -541,8 +544,10 @@ public void testPipelineExclusionWithPipelineFailure() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestHelper - .createKey(keyName, type, size, objectStore, volumeName, bucketName); + return TestDataUtil + .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), size); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index b5677f5cfc73..25003056dd94 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -34,6 +34,8 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -52,6 +54,7 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -229,8 +232,10 @@ public void testPipelineExclusionWithPipelineFailure() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestHelper - .createKey(keyName, type, size, objectStore, volumeName, bucketName); + return TestDataUtil + .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), size); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java index 86d568b7d94f..95d54df99ad2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java @@ -29,6 +29,8 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -42,6 +44,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -233,8 +236,10 @@ public void testMultiBlockWritesWithIntermittentDnFailures() private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestHelper - .createKey(keyName, type, size, objectStore, volumeName, bucketName); + return TestDataUtil + .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), size); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 16e20aeaf552..3f267c3cf23b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.UUID; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -43,6 +44,7 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -183,9 +185,10 @@ public void testGroupMismatchExceptionHandling() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestHelper - .createKey(keyName, type, ReplicationFactor.ONE, - size, objectStore, volumeName, bucketName); + return TestDataUtil + .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), size); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index 9148282cd9af..a5b5301109cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -253,9 +255,10 @@ void testMaxRetriesByOzoneClient() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestHelper - .createKey(keyName, type, ReplicationFactor.ONE, - size, objectStore, volumeName, bucketName); + return TestDataUtil + .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), size); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java index c454facea070..dcd4b3897bba 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java @@ -132,22 +132,6 @@ public static int countReplicas(long containerID, return count; } - public static OzoneOutputStream createKey(String keyName, - ReplicationType type, long size, ObjectStore objectStore, - String volumeName, String bucketName) throws Exception { - if (type == ReplicationType.STAND_ALONE) { - throw new IllegalArgumentException(ReplicationType.STAND_ALONE + - " replication type should not be used in tests to write keys anymore." - ); - } - org.apache.hadoop.hdds.client.ReplicationFactor factor = - org.apache.hadoop.hdds.client.ReplicationFactor.THREE; - ReplicationConfig config = - ReplicationConfig.fromTypeAndFactor(type, factor); - return objectStore.getVolume(volumeName).getBucket(bucketName) - .createKey(keyName, size, config, new HashMap<>()); - } - public static OzoneDataStreamOutput createStreamKey(String keyName, ReplicationType type, long size, ObjectStore objectStore, String volumeName, String bucketName) throws Exception { @@ -161,24 +145,6 @@ public static OzoneDataStreamOutput createStreamKey(String keyName, .createStreamKey(keyName, size, config, new HashMap<>()); } - public static OzoneOutputStream createKey(String keyName, - ReplicationType type, - org.apache.hadoop.hdds.client.ReplicationFactor factor, long size, - ObjectStore objectStore, String volumeName, String bucketName) - throws Exception { - ReplicationConfig config = - ReplicationConfig.fromTypeAndFactor(type, factor); - return objectStore.getVolume(volumeName).getBucket(bucketName) - .createKey(keyName, size, config, new HashMap<>()); - } - - public static OzoneOutputStream createKey(String keyName, - ReplicationConfig replicationConfig, long size, ObjectStore objectStore, - String volumeName, String bucketName) throws Exception { - return objectStore.getVolume(volumeName).getBucket(bucketName) - .createKey(keyName, size, replicationConfig, new HashMap<>()); - } - public static void validateData(String keyName, byte[] data, ObjectStore objectStore, String volumeName, String bucketName) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java index b9710b45455d..4006037a066d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java @@ -19,7 +19,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; -import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; import static org.assertj.core.api.Assertions.assertThat; @@ -47,6 +46,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -56,6 +57,7 @@ import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -221,8 +223,9 @@ protected void readFromCorruptedKey(String keyName) throws IOException { } private OzoneOutputStream createKey(String keyName) throws Exception { - return TestHelper.createKey( - keyName, RATIS, ONE, 0, store, volumeName, bucketName); + return TestDataUtil.createKey(store.getVolume(volumeName).getBucket(bucketName), + keyName, ReplicationConfig. + fromTypeAndFactor(ReplicationType.RATIS, ONE), 0); } /** From 86231bc476ea2d472b0b4781a95f223cd49c1587 Mon Sep 17 00:00:00 2001 From: Chia-Chuan Yu Date: Sun, 6 Apr 2025 13:48:42 +0800 Subject: [PATCH 4/5] Fixed styles --- .../client/rpc/TestBlockOutputStream.java | 1 - .../recon/TestReconWithOzoneManagerFSO.java | 18 ++++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index bf7ec52e99f5..b5a22128afe2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -59,7 +59,6 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index b8774cb00996..41c8766c9f7d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.container.ContainerTestHelper; @@ -82,23 +83,16 @@ public static void shutdown() { } } - private void writeTestData(String volumeName, - String bucketName, - String keyName) throws Exception { - - String keyString = UUID.randomUUID().toString(); - byte[] data = ContainerTestHelper.getFixedLengthString( - keyString, 100).getBytes(UTF_8); - TestDataUtil.createKey(store.getVolume(volumeName). - getBucket(bucketName), keyName, data); - } - private void writeKeys(String vol, String bucket, String key) throws Exception { store.createVolume(vol); OzoneVolume volume = store.getVolume(vol); volume.createBucket(bucket); - writeTestData(vol, bucket, key); + OzoneBucket ozoneBucket = volume.getBucket(bucket); + String keyString = UUID.randomUUID().toString(); + byte[] data = ContainerTestHelper.getFixedLengthString( + keyString, 100).getBytes(UTF_8); + TestDataUtil.createKey(ozoneBucket, key, data); } @Test From c5068c856bf2bf90b552f2bdcbdc16797be0de11 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 6 Apr 2025 12:13:54 +0200 Subject: [PATCH 5/5] Revert "Refactor the testHelp.createkey()" This reverts commit 1d2e401cb64fbb7209c24b6bb68cf657fc55db7c. --- .../hadoop/hdds/scm/TestWatchForCommit.java | 14 ++++---- .../org/apache/hadoop/ozone/TestDataUtil.java | 6 ---- .../client/rpc/TestBlockOutputStream.java | 8 ++--- .../TestCloseContainerHandlingByClient.java | 9 ++--- .../rpc/TestDiscardPreallocatedBlocks.java | 8 ++--- .../client/rpc/TestECKeyOutputStream.java | 19 +++++------ .../rpc/TestFailureHandlingByClient.java | 9 ++--- ...TestFailureHandlingByClientFlushDelay.java | 9 ++--- .../TestMultiBlockWritesWithDnFailures.java | 9 ++--- ...oneClientRetriesOnExceptionFlushDelay.java | 9 ++--- .../TestOzoneClientRetriesOnExceptions.java | 9 ++--- .../hadoop/ozone/container/TestHelper.java | 34 +++++++++++++++++++ ...stContainerScannerIntegrationAbstract.java | 9 ++--- 13 files changed, 72 insertions(+), 80 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java index 7e9e424c36f6..136a640d7c33 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestWatchForCommit.java @@ -19,7 +19,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; -import static org.apache.hadoop.ozone.TestDataUtil.createKey; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -36,8 +35,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -183,10 +180,7 @@ private String getKeyName() { @Test public void testWatchForCommitWithKeyWrite() throws Exception { String keyName = getKeyName(); - - OzoneOutputStream key = createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), 0); + OzoneOutputStream key = createKey(keyName, ReplicationType.RATIS, 0); int dataLength = maxFlushSize + 50; // write data more than 1 chunk byte[] data1 = @@ -398,6 +392,12 @@ public void testWatchForCommitForGroupMismatchException() throws Exception { } } + private OzoneOutputStream createKey(String keyName, ReplicationType type, + long size) throws Exception { + return TestHelper + .createKey(keyName, type, size, objectStore, volumeName, bucketName); + } + private void validateData(String keyName, byte[] data) throws Exception { TestHelper .validateData(keyName, data, objectStore, volumeName, bucketName); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index a58668b20001..ab6e7de9c735 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -43,7 +43,6 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.VolumeArgs; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -135,11 +134,6 @@ public static void createKey(OzoneBucket bucket, String keyName, } } - public static OzoneOutputStream createKey(OzoneBucket bucket, String keyName, - ReplicationConfig replicationConfig, long size) throws Exception { - return bucket.createKey(keyName, size, replicationConfig, new HashMap<>()); - } - public static String getKey(OzoneBucket bucket, String keyName) throws IOException { try (InputStream stream = bucket.readKey(keyName)) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index b5a22128afe2..a67dca9e8eaf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -35,7 +35,6 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Stream; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -53,12 +52,12 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.container.TestHelper; import org.apache.ozone.test.tag.Flaky; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -780,9 +779,8 @@ static OzoneOutputStream createKey(OzoneClient client, String keyName) static OzoneOutputStream createKey(OzoneClient client, String keyName, long size, ReplicationFactor factor) throws Exception { - ReplicationConfig config = ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, factor); - return TestDataUtil.createKey(client.getObjectStore().getVolume(VOLUME).getBucket(BUCKET), - keyName, config, size); + return TestHelper.createKey(keyName, ReplicationType.RATIS, factor, size, + client.getObjectStore(), VOLUME, BUCKET); } static String getKeyName() { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index f2ed50fae052..26f596c44335 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -31,13 +31,10 @@ import java.util.UUID; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -324,10 +321,8 @@ private void waitForContainerClose(OzoneOutputStream outputStream) private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestDataUtil - .createKey(objectStore. - getVolume(volumeName).getBucket(bucketName), keyName, - ReplicationConfig.fromTypeAndFactor(type, ReplicationFactor.THREE), size); + return TestHelper + .createKey(keyName, type, size, objectStore, volumeName, bucketName); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java index 94ad7bb134f3..ce78960918cc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java @@ -20,7 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; -import static org.apache.hadoop.ozone.TestDataUtil.createKey; +import static org.apache.hadoop.ozone.container.TestHelper.createKey; import static org.apache.hadoop.ozone.container.TestHelper.waitForContainerClose; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -30,8 +30,6 @@ import java.util.List; import java.util.UUID; import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -91,9 +89,7 @@ void cleanup() { public void testDiscardPreallocatedBlocks() throws Exception { String keyName = "key"; OzoneOutputStream key = - createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), 2L * blockSize); + createKey(keyName, ReplicationType.RATIS, 2L * blockSize, objectStore, volumeName, bucketName); KeyOutputStream keyOutputStream = assertInstanceOf(KeyOutputStream.class, key.getOutputStream()); // With the initial size provided, it should have pre allocated 2 blocks diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java index 9382dc86f766..2bc5947a7000 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java @@ -61,7 +61,6 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -74,6 +73,7 @@ import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.ozone.test.GenericTestUtils; @@ -181,10 +181,10 @@ public static void shutdown() { @Test public void testCreateKeyWithECReplicationConfig() throws Exception { - try (OzoneOutputStream key = TestDataUtil - .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyString, new ECReplicationConfig(3, 2, - ECReplicationConfig.EcCodec.RS, chunkSize), inputSize)) { + try (OzoneOutputStream key = TestHelper + .createKey(keyString, new ECReplicationConfig(3, 2, + ECReplicationConfig.EcCodec.RS, chunkSize), inputSize, + objectStore, volumeName, bucketName)) { assertInstanceOf(ECKeyOutputStream.class, key.getOutputStream()); } } @@ -212,9 +212,8 @@ public void testECKeyCreatetWithDatanodeIdChange() ObjectStore store = client1.getObjectStore(); store.createVolume(volumeName); store.getVolume(volumeName).createBucket(bucketName); - OzoneOutputStream key = TestDataUtil.createKey(store.getVolume(volumeName).getBucket(bucketName), - keyString, new ECReplicationConfig(3, 2, - ECReplicationConfig.EcCodec.RS, 1024), inputSize); + OzoneOutputStream key = TestHelper.createKey(keyString, new ECReplicationConfig(3, 2, + ECReplicationConfig.EcCodec.RS, 1024), inputSize, store, volumeName, bucketName); byte[] b = new byte[6 * 1024]; ECKeyOutputStream groupOutputStream = (ECKeyOutputStream) key.getOutputStream(); List locationInfoList = groupOutputStream.getLocationInfoList(); @@ -574,9 +573,9 @@ private byte[] getInputBytes(int offset, int bufferChunks, int numChunks) { @Test public void testBlockedHflushAndHsync() throws Exception { // Expect ECKeyOutputStream hflush and hsync calls to throw exception - try (OzoneOutputStream oOut = TestDataUtil.createKey(objectStore.getVolume(volumeName).getBucket(bucketName), + try (OzoneOutputStream oOut = TestHelper.createKey( keyString, new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, chunkSize), - inputSize)) { + inputSize, objectStore, volumeName, bucketName)) { assertInstanceOf(ECKeyOutputStream.class, oOut.getOutputStream()); KeyOutputStream kOut = (KeyOutputStream) oOut.getOutputStream(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java index c09584e3582d..30446af41b0e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java @@ -36,8 +36,6 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -55,7 +53,6 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -544,10 +541,8 @@ public void testPipelineExclusionWithPipelineFailure() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestDataUtil - .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), size); + return TestHelper + .createKey(keyName, type, size, objectStore, volumeName, bucketName); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index 25003056dd94..b5677f5cfc73 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -34,8 +34,6 @@ import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -54,7 +52,6 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -232,10 +229,8 @@ public void testPipelineExclusionWithPipelineFailure() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestDataUtil - .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), size); + return TestHelper + .createKey(keyName, type, size, objectStore, volumeName, bucketName); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java index 95d54df99ad2..86d568b7d94f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestMultiBlockWritesWithDnFailures.java @@ -29,8 +29,6 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -44,7 +42,6 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -236,10 +233,8 @@ public void testMultiBlockWritesWithIntermittentDnFailures() private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestDataUtil - .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), size); + return TestHelper + .createKey(keyName, type, size, objectStore, volumeName, bucketName); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 3f267c3cf23b..16e20aeaf552 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.UUID; -import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -44,7 +43,6 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -185,10 +183,9 @@ public void testGroupMismatchExceptionHandling() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestDataUtil - .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), size); + return TestHelper + .createKey(keyName, type, ReplicationFactor.ONE, + size, objectStore, volumeName, bucketName); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index a5b5301109cf..9148282cd9af 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -49,7 +48,6 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -255,10 +253,9 @@ void testMaxRetriesByOzoneClient() throws Exception { private OzoneOutputStream createKey(String keyName, ReplicationType type, long size) throws Exception { - return TestDataUtil - .createKey(objectStore.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), size); + return TestHelper + .createKey(keyName, type, ReplicationFactor.ONE, + size, objectStore, volumeName, bucketName); } private void validateData(String keyName, byte[] data) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java index dcd4b3897bba..c454facea070 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java @@ -132,6 +132,22 @@ public static int countReplicas(long containerID, return count; } + public static OzoneOutputStream createKey(String keyName, + ReplicationType type, long size, ObjectStore objectStore, + String volumeName, String bucketName) throws Exception { + if (type == ReplicationType.STAND_ALONE) { + throw new IllegalArgumentException(ReplicationType.STAND_ALONE + + " replication type should not be used in tests to write keys anymore." + ); + } + org.apache.hadoop.hdds.client.ReplicationFactor factor = + org.apache.hadoop.hdds.client.ReplicationFactor.THREE; + ReplicationConfig config = + ReplicationConfig.fromTypeAndFactor(type, factor); + return objectStore.getVolume(volumeName).getBucket(bucketName) + .createKey(keyName, size, config, new HashMap<>()); + } + public static OzoneDataStreamOutput createStreamKey(String keyName, ReplicationType type, long size, ObjectStore objectStore, String volumeName, String bucketName) throws Exception { @@ -145,6 +161,24 @@ public static OzoneDataStreamOutput createStreamKey(String keyName, .createStreamKey(keyName, size, config, new HashMap<>()); } + public static OzoneOutputStream createKey(String keyName, + ReplicationType type, + org.apache.hadoop.hdds.client.ReplicationFactor factor, long size, + ObjectStore objectStore, String volumeName, String bucketName) + throws Exception { + ReplicationConfig config = + ReplicationConfig.fromTypeAndFactor(type, factor); + return objectStore.getVolume(volumeName).getBucket(bucketName) + .createKey(keyName, size, config, new HashMap<>()); + } + + public static OzoneOutputStream createKey(String keyName, + ReplicationConfig replicationConfig, long size, ObjectStore objectStore, + String volumeName, String bucketName) throws Exception { + return objectStore.getVolume(volumeName).getBucket(bucketName) + .createKey(keyName, size, replicationConfig, new HashMap<>()); + } + public static void validateData(String keyName, byte[] data, ObjectStore objectStore, String volumeName, String bucketName) throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java index 4006037a066d..b9710b45455d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestContainerScannerIntegrationAbstract.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; +import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; import static org.assertj.core.api.Assertions.assertThat; @@ -46,8 +47,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -57,7 +56,6 @@ import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -223,9 +221,8 @@ protected void readFromCorruptedKey(String keyName) throws IOException { } private OzoneOutputStream createKey(String keyName) throws Exception { - return TestDataUtil.createKey(store.getVolume(volumeName).getBucket(bucketName), - keyName, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ONE), 0); + return TestHelper.createKey( + keyName, RATIS, ONE, 0, store, volumeName, bucketName); } /**