diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java index 0b8449cb08db..381c86d403aa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java @@ -35,9 +35,6 @@ import java.time.Duration; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -109,8 +106,7 @@ public void testReplicasAreReportedForClosedContainerAfterRestart() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), + TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(StandardCharsets.UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); @@ -155,8 +151,7 @@ public void testCloseClosedContainer() throws Exception { // Create some keys to write data into the open containers for (int i = 0; i < 10; i++) { - TestDataUtil.createKey(bucket, "key" + i, ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), + TestDataUtil.createKey(bucket, "key" + i, "this is the content".getBytes(StandardCharsets.UTF_8)); } StorageContainerManager scm = cluster.getStorageContainerManager(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index b54bcd6f624b..8215c7a5a426 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -106,18 +106,23 @@ public static OzoneVolume createVolume(OzoneClient client, public static void createKey(OzoneBucket bucket, String keyName, byte[] content) throws IOException { - ReplicationConfig replicationConfig = ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE); - createKey(bucket, keyName, replicationConfig, content); + createKey(bucket, keyName, null, content); } + public static OutputStream createOutputStream(OzoneBucket bucket, String keyName, + ReplicationConfig repConfig, byte[] content) + throws IOException { + return repConfig == null + ? bucket.createKey(keyName, content.length) + : bucket.createKey(keyName, content.length, repConfig, new HashMap<>()); + } + public static void createKey(OzoneBucket bucket, String keyName, ReplicationConfig repConfig, byte[] content) throws IOException { - try (OutputStream stream = bucket - .createKey(keyName, content.length, repConfig, - new HashMap<>())) { + try (OutputStream stream = createOutputStream(bucket, keyName, + repConfig, content)) { stream.write(content); } } @@ -214,7 +219,9 @@ public static Map createKeys(MiniOzoneCluster cluster, int nu OzoneBucket bucket = createVolumeAndBucket(client); for (int i = 0; i < numOfKeys; i++) { String keyName = RandomStringUtils.randomAlphabetic(5) + i; - createKey(bucket, keyName, RandomStringUtils.randomAlphabetic(5).getBytes(UTF_8)); + createKey(bucket, keyName, ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + RandomStringUtils.randomAlphabetic(5).getBytes(UTF_8)); keyLocationMap.put(keyName, lookupOmKeyInfo(cluster, bucket, keyName)); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java index 2d6fbd1cfbc2..2f98caeb6965 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java @@ -34,8 +34,6 @@ import java.util.Optional; import java.util.stream.Stream; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.TestDataUtil; @@ -377,8 +375,7 @@ private static void createAndAssertKeys(OzoneBucket ozoneBucket, List ke byte[] input = new byte[length]; Arrays.fill(input, (byte) 96); for (String key : keys) { - createKey(ozoneBucket, key, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), input); + createKey(ozoneBucket, key, input); // Read the key with given key name. readkey(ozoneBucket, key, length, input); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java index 1f69f647c3ce..c8e8c21995ce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeysWithFSO.java @@ -31,8 +31,6 @@ import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.utils.IOUtils; @@ -649,8 +647,7 @@ private static void createAndAssertKeys(OzoneBucket ozoneBucket, List ke byte[] input = new byte[length]; Arrays.fill(input, (byte) 96); for (String key : keys) { - createKey(ozoneBucket, key, ReplicationConfig - .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), input); + createKey(ozoneBucket, key, input); // Read the key with given key name. readkey(ozoneBucket, key, length, input); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java index 076bf2abd7a6..75a6004643d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java @@ -98,6 +98,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -676,9 +679,11 @@ private void prepSnapshotData() throws Exception { .createVolumeAndBucket(client); // Create dummy keys for snapshotting. - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "content".getBytes(StandardCharsets.UTF_8)); - TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), + TestDataUtil.createKey(bucket, UUID.randomUUID().toString(), ReplicationConfig + .fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), "content".getBytes(StandardCharsets.UTF_8)); snapshotDirName = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java index be7a929a83de..87dd31c2178b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java @@ -49,9 +49,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import org.apache.commons.compress.utils.Lists; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; @@ -202,10 +199,8 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { OzoneBucket bucket2 = TestDataUtil.createBucket( client, VOLUME_NAME, bucketArgs, BUCKET_NAME_TWO); // Create key1 and key2 - TestDataUtil.createKey(bucket2, "bucket2key1", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket2, "bucket2key2", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket2, "bucket2key1", CONTENT.array()); + TestDataUtil.createKey(bucket2, "bucket2key2", CONTENT.array()); // Create Snapshot client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, @@ -265,16 +260,12 @@ public void testSnapshotWithFSO() throws Exception { // Create 10 keys for (int i = 1; i <= 10; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Create 5 keys to overwrite for (int i = 11; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Create Directory and Sub @@ -287,8 +278,7 @@ public void testSnapshotWithFSO() throws Exception { String childDir = "/childDir" + j; client.getProxy().createDirectory(VOLUME_NAME, BUCKET_NAME_FSO, parent + childDir); - TestDataUtil.createKey(bucket2, parent + childFile, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket2, parent + childFile, CONTENT.array()); } } @@ -304,8 +294,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 3 keys -> Moves previous version to deletedTable for (int i = 11; i <= 13; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } assertTableRowCount(keyTable, 24); @@ -369,9 +358,7 @@ public void testSnapshotWithFSO() throws Exception { // Overwrite 2 keys for (int i = 14; i <= 15; i++) { - TestDataUtil.createKey(bucket2, "key" + i, ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket2, "key" + i, CONTENT.array()); } // Delete 2 more keys @@ -727,12 +714,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) om.getMetadataManager(); - TestDataUtil.createKey(bucket, bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key1", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key1", CONTENT.array()); assertTableRowCount(keyTable, 2); // Create Snapshot 1. @@ -742,10 +725,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key2", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key2", CONTENT.array()); // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), @@ -769,10 +750,8 @@ private synchronized void createSnapshotDataForBucket(OzoneBucket bucket) throws // deletedTable when Snapshot 2 is taken. assertTableRowCount(deletedTable, 0); - TestDataUtil.createKey(bucket, bucket.getName() + "key3", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, bucket.getName() + "key4", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key3", CONTENT.array()); + TestDataUtil.createKey(bucket, bucket.getName() + "key4", CONTENT.array()); client.getProxy().deleteKey(bucket.getVolumeName(), bucket.getName(), bucket.getName() + "key4", false); assertTableRowCount(keyTable, 1); @@ -832,19 +811,15 @@ private synchronized void createSnapshotFSODataForBucket(OzoneBucket bucket) thr throw new RuntimeException(ex); } })); - TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, "dir1/" + bucket.getName() + "key1", CONTENT.array()); assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 2); assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 2); // Overwrite bucket1key0, This is a newer version of the key which should // reclaimed as this is a different version of the key. - TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); - TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", ReplicationConfig. - fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), CONTENT.array()); + TestDataUtil.createKey(bucket, "dir0/" + bucket.getName() + "key0", CONTENT.array()); + TestDataUtil.createKey(bucket, "dir2/" + bucket.getName() + "key2", CONTENT.array()); assertTableRowCount(keyTable, countMap.get(keyTable.getName()) + 3); assertTableRowCount(dirTable, countMap.get(dirTable.getName()) + 3); assertTableRowCount(deletedTable, countMap.get(deletedTable.getName()) + 1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java index e6ada443ccd4..75ddd145d96d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneContainerUpgradeShell.java @@ -42,9 +42,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -183,8 +180,7 @@ private static void writeKey(String keyName) throws IOException { TestDataUtil.createVolumeAndBucket(client, VOLUME_NAME, BUCKET_NAME); TestDataUtil.createKey( client.getObjectStore().getVolume(VOLUME_NAME).getBucket(BUCKET_NAME), - keyName, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.THREE), - "test".getBytes(StandardCharsets.UTF_8)); + keyName, "test".getBytes(StandardCharsets.UTF_8)); } }