From d45e83c10dddba09b3ede3c9921b1f0dc7e5b167 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 12 Feb 2025 10:52:08 +0100 Subject: [PATCH 1/5] convert TestCloseContainerHandlingByClient --- .../TestCloseContainerHandlingByClient.java | 80 ++++++------------- .../org/apache/ozone/test/NonHATests.java | 9 +++ 2 files changed, 34 insertions(+), 55 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index fde156ad15e0..26f596c44335 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -20,32 +20,24 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.UUID; -import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneInputStream; @@ -55,56 +47,40 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; /** * Tests Close Container Exception handling by Ozone Client. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) @Timeout(300) -public class TestCloseContainerHandlingByClient { - - private static MiniOzoneCluster cluster; - private static OzoneConfiguration conf = new OzoneConfiguration(); - private static OzoneClient client; - private static ObjectStore objectStore; - private static int chunkSize; - private static int blockSize; - private static String volumeName; - private static String bucketName; - private static String keyString; - - /** - * Create a MiniDFSCluster for testing. - *

- * Ozone is made active by setting OZONE_ENABLED = true - * - * @throws IOException - */ +public abstract class TestCloseContainerHandlingByClient implements NonHATests.TestCase { + + private MiniOzoneCluster cluster; + private OzoneClient client; + private ObjectStore objectStore; + private int chunkSize; + private int blockSize; + private String volumeName; + private String bucketName; + private String keyString; + @BeforeAll - public static void init() throws Exception { - chunkSize = (int) OzoneConsts.MB; - blockSize = 4 * chunkSize; - - OzoneClientConfig config = conf.getObject(OzoneClientConfig.class); - config.setChecksumType(ChecksumType.NONE); - conf.setFromObject(config); - - conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS); - conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 1); - conf.setQuietMode(false); - conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, - StorageUnit.MB); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build(); - cluster.waitForClusterToBeReady(); - //the easiest way to create an open container is creating a key - client = OzoneClientFactory.getRpcClient(conf); + void init() throws Exception { + chunkSize = (int) cluster().getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, 1024 * 1024, StorageUnit.BYTES); + blockSize = (int) cluster().getConf().getStorageSize(OZONE_SCM_BLOCK_SIZE, 4 * chunkSize, StorageUnit.BYTES); + + cluster = cluster(); + client = cluster().newClient(); objectStore = client.getObjectStore(); keyString = UUID.randomUUID().toString(); - volumeName = "closecontainerexceptionhandlingtest"; - bucketName = volumeName; + volumeName = "vol-" + UUID.randomUUID(); + bucketName = "bucket"; objectStore.createVolume(volumeName); objectStore.getVolume(volumeName).createBucket(bucketName); } @@ -113,15 +89,9 @@ private String getKeyName() { return UUID.randomUUID().toString(); } - /** - * Shutdown MiniDFSCluster. - */ @AfterAll - public static void shutdown() { + void cleanup() { IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java index 33a12bfadc0f..59236cb87966 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestCpuMetrics; import org.apache.hadoop.ozone.admin.om.lease.TestLeaseRecoverer; +import org.apache.hadoop.ozone.client.rpc.TestCloseContainerHandlingByClient; import org.apache.hadoop.ozone.client.rpc.TestDiscardPreallocatedBlocks; import org.apache.hadoop.ozone.om.TestBucketLayoutWithOlderClient; import org.apache.hadoop.ozone.om.TestListKeys; @@ -214,6 +215,14 @@ public MiniOzoneCluster cluster() { } } + @Nested + class CloseContainerHandlingByClient extends TestCloseContainerHandlingByClient { + @Override + public MiniOzoneCluster cluster() { + return getCluster(); + } + } + @Nested class CpuMetrics extends TestCpuMetrics { @Override From e3ddf6a3bc7f083b25efe4857b478f17addf4226 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 20 Feb 2025 12:33:11 +0100 Subject: [PATCH 2/5] convert TestContainerStateMachineStream --- .../rpc/TestContainerStateMachineStream.java | 206 +++++------------- .../org/apache/ozone/test/NonHATests.java | 9 + 2 files changed, 58 insertions(+), 157 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java index ee88fca53a63..63c2eaab0fda 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java @@ -17,200 +17,92 @@ package org.apache.hadoop.ozone.client.rpc; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.IOException; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.ozone.container.TestHelper.createStreamKey; +import static org.apache.hadoop.ozone.container.TestHelper.getDatanodeService; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + import java.nio.ByteBuffer; -import java.time.Duration; import java.util.List; -import java.util.concurrent.TimeUnit; +import java.util.UUID; +import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigForTesting; -import org.apache.hadoop.ozone.HddsDatanodeService; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.io.KeyDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.ozone.container.TestHelper; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.apache.ozone.test.NonHATests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Tests the containerStateMachine stream handling. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) @Timeout(300) -public class TestContainerStateMachineStream { - private MiniOzoneCluster cluster; - private OzoneConfiguration conf = new OzoneConfiguration(); +public abstract class TestContainerStateMachineStream implements NonHATests.TestCase { private OzoneClient client; private ObjectStore objectStore; private String volumeName; private String bucketName; + private int chunkSize; + + @BeforeAll + void setup() throws Exception { + chunkSize = (int) cluster().getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, 1024 * 1024, StorageUnit.BYTES); - private static final int CHUNK_SIZE = 100; - private static final int FLUSH_SIZE = 2 * CHUNK_SIZE; - private static final int MAX_FLUSH_SIZE = 2 * FLUSH_SIZE; - private static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE; - - /** - * Create a MiniDFSCluster for testing. - * - * @throws IOException - */ - @BeforeEach - public void setup() throws Exception { - conf = new OzoneConfiguration(); - - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setStreamBufferFlushDelay(false); - conf.setFromObject(clientConfig); - - conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, - TimeUnit.MILLISECONDS); - conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200, - TimeUnit.MILLISECONDS); - conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 200, TimeUnit.MILLISECONDS); - conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 200, - TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - conf.setTimeDuration(OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, 1, - TimeUnit.SECONDS); - - RatisClientConfig ratisClientConfig = - conf.getObject(RatisClientConfig.class); - ratisClientConfig.setWriteRequestTimeout(Duration.ofSeconds(10)); - ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(10)); - conf.setFromObject(ratisClientConfig); - - DatanodeRatisServerConfig ratisServerConfig = - conf.getObject(DatanodeRatisServerConfig.class); - ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); - ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10)); - conf.setFromObject(ratisServerConfig); - - RatisClientConfig.RaftConfig raftClientConfig = - conf.getObject(RatisClientConfig.RaftConfig.class); - raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3)); - raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); - conf.setFromObject(raftClientConfig); - - ClientConfigForTesting.newBuilder(StorageUnit.BYTES) - .setDataStreamMinPacketSize(1024) - .setBlockSize(BLOCK_SIZE) - .setChunkSize(CHUNK_SIZE) - .setStreamBufferFlushSize(FLUSH_SIZE) - .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .applyTo(conf); - - conf.setLong(OzoneConfigKeys.HDDS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); - conf.setQuietMode(false); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .build(); - cluster.waitForClusterToBeReady(); - cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 60000); - //the easiest way to create an open container is creating a key - client = OzoneClientFactory.getRpcClient(conf); + client = cluster().newClient(); objectStore = client.getObjectStore(); - volumeName = "testcontainerstatemachinestream"; + volumeName = "vol-" + UUID.randomUUID(); bucketName = "teststreambucket"; objectStore.createVolume(volumeName); objectStore.getVolume(volumeName).createBucket(bucketName); - } - /** - * Shutdown MiniDFSCluster. - */ - @AfterEach - public void shutdown() { + @AfterAll + void shutdown() { IOUtils.closeQuietly(client); - if (cluster != null) { - cluster.shutdown(); - } - } - - @Test - public void testContainerStateMachineForStreaming() throws Exception { - long size = CHUNK_SIZE + 1; - - OzoneDataStreamOutput key = TestHelper.createStreamKey( - "ozone-stream-test.txt", ReplicationType.RATIS, size, objectStore, - volumeName, bucketName); - - byte[] data = ContainerTestHelper.generateData((int) size, true); - key.write(ByteBuffer.wrap(data)); - key.flush(); - - KeyDataStreamOutput streamOutput = - (KeyDataStreamOutput) key.getByteBufStreamOutput(); - List locationInfoList = - streamOutput.getLocationInfoList(); - - key.close(); - - OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0); - HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo, - cluster); - - long bytesUsed = dn.getDatanodeStateMachine() - .getContainer().getContainerSet() - .getContainer(omKeyLocationInfo.getContainerID()). - getContainerData().getBytesUsed(); - - assertEquals(bytesUsed, size); } + @ParameterizedTest + @ValueSource(ints = {-1, +1}) + void testContainerStateMachineForStreaming(int offset) throws Exception { + final int size = chunkSize + offset; - @Test - public void testContainerStateMachineForStreamingSmallFile() - throws Exception { - long size = CHUNK_SIZE - 1; + final List locationInfoList; + try (OzoneDataStreamOutput key = createStreamKey("key" + offset, ReplicationType.RATIS, size, + objectStore, volumeName, bucketName)) { - OzoneDataStreamOutput key = TestHelper.createStreamKey( - "ozone-stream-test-small-file.txt", ReplicationType.RATIS, size, - objectStore, volumeName, bucketName); + byte[] data = ContainerTestHelper.generateData(size, true); + key.write(ByteBuffer.wrap(data)); + key.flush(); - byte[] data = ContainerTestHelper.generateData((int) size, true); - key.write(ByteBuffer.wrap(data)); - key.flush(); + locationInfoList = assertInstanceOf(KeyDataStreamOutput.class, key.getByteBufStreamOutput()) + .getLocationInfoList(); + } - KeyDataStreamOutput streamOutput = - (KeyDataStreamOutput) key.getByteBufStreamOutput(); - List locationInfoList = - streamOutput.getLocationInfoList(); - key.close(); OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0); - HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo, - cluster); - - long bytesUsed = dn.getDatanodeStateMachine() - .getContainer().getContainerSet() - .getContainer(omKeyLocationInfo.getContainerID()). - getContainerData().getBytesUsed(); - assertEquals(bytesUsed, size); + long bytesUsed = getDatanodeService(omKeyLocationInfo, cluster()) + .getDatanodeStateMachine() + .getContainer() + .getContainerSet() + .getContainer(omKeyLocationInfo.getContainerID()) + .getContainerData() + .getBytesUsed(); + + assertThat(bytesUsed) + // container may have previous data + .isGreaterThanOrEqualTo(size); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java index 59236cb87966..91fdcb4576d4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.TestCpuMetrics; import org.apache.hadoop.ozone.admin.om.lease.TestLeaseRecoverer; import org.apache.hadoop.ozone.client.rpc.TestCloseContainerHandlingByClient; +import org.apache.hadoop.ozone.client.rpc.TestContainerStateMachineStream; import org.apache.hadoop.ozone.client.rpc.TestDiscardPreallocatedBlocks; import org.apache.hadoop.ozone.om.TestBucketLayoutWithOlderClient; import org.apache.hadoop.ozone.om.TestListKeys; @@ -223,6 +224,14 @@ public MiniOzoneCluster cluster() { } } + @Nested + class ContainerStateMachineStream extends TestContainerStateMachineStream { + @Override + public MiniOzoneCluster cluster() { + return getCluster(); + } + } + @Nested class CpuMetrics extends TestCpuMetrics { @Override From 5ce9e8c955f91f88be002dde3c9019eb2a586d45 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 13 Mar 2025 14:12:18 +0100 Subject: [PATCH 3/5] convert TestOzoneRpcClientWithKeyLatestVersion --- ...estOzoneRpcClientWithKeyLatestVersion.java | 28 ++----------------- .../org/apache/ozone/test/NonHATests.java | 9 ++++++ 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java index 453af62bfd80..50ea1c0eee4a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.client.rpc; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,7 +34,6 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -44,8 +42,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -54,33 +51,14 @@ * Main purpose of this test is with OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION * set/unset key create/read works properly or not for buckets * with/without versioning. - * TODO: can be merged with other test class */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class TestOzoneRpcClientWithKeyLatestVersion { - - private MiniOzoneCluster cluster; - - @BeforeAll - void setup() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); - cluster = MiniOzoneCluster.newBuilder(conf) - .build(); - cluster.waitForClusterToBeReady(); - } - - @AfterAll - void tearDown() { - if (cluster != null) { - cluster.shutdown(); - } - } +public abstract class TestOzoneRpcClientWithKeyLatestVersion implements NonHATests.TestCase { @ParameterizedTest @ValueSource(booleans = {true, false}) void testWithGetLatestVersion(boolean getLatestVersionOnly) throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(cluster.getConf()); + OzoneConfiguration conf = new OzoneConfiguration(cluster().getConf()); conf.setBoolean(OZONE_CLIENT_KEY_LATEST_VERSION_LOCATION, getLatestVersionOnly); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java index 91fdcb4576d4..44fde35bc8a0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.client.rpc.TestCloseContainerHandlingByClient; import org.apache.hadoop.ozone.client.rpc.TestContainerStateMachineStream; import org.apache.hadoop.ozone.client.rpc.TestDiscardPreallocatedBlocks; +import org.apache.hadoop.ozone.client.rpc.TestOzoneRpcClientWithKeyLatestVersion; import org.apache.hadoop.ozone.om.TestBucketLayoutWithOlderClient; import org.apache.hadoop.ozone.om.TestListKeys; import org.apache.hadoop.ozone.om.TestListKeysWithFSO; @@ -336,6 +337,14 @@ public MiniOzoneCluster cluster() { } } + @Nested + class OzoneRpcClientWithKeyLatestVersion extends TestOzoneRpcClientWithKeyLatestVersion { + @Override + public MiniOzoneCluster cluster() { + return getCluster(); + } + } + @Nested class DatanodeReconfiguration extends TestDatanodeReconfiguration { @Override From 8f947f52fc123fe923d0a744f4e4833cb3fc9153 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 20 Mar 2025 19:20:57 +0100 Subject: [PATCH 4/5] convert TestOzoneClientMultipartUploadWithFSO --- ...TestOzoneClientMultipartUploadWithFSO.java | 76 ++++++------------- .../org/apache/ozone/test/NonHATests.java | 9 +++ 2 files changed, 31 insertions(+), 54 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index c953a521c78f..8e6a74f838b2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -56,8 +55,8 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneTestUtils; @@ -65,7 +64,6 @@ import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneKeyLocation; import org.apache.hadoop.ozone.client.OzoneMultipartUpload; @@ -75,6 +73,7 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -86,12 +85,13 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; -import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -99,70 +99,38 @@ /** * This test verifies all the S3 multipart client apis - prefix layout. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) @Timeout(300) -public class TestOzoneClientMultipartUploadWithFSO { +public abstract class TestOzoneClientMultipartUploadWithFSO implements NonHATests.TestCase { - private static ObjectStore store = null; - private static MiniOzoneCluster cluster = null; - private static OzoneClient ozClient = null; - private static MessageDigest eTagProvider; + private ObjectStore store; + private MiniOzoneCluster cluster; + private OzoneClient ozClient; + private MessageDigest eTagProvider; private String volumeName; private String bucketName; private String keyName; private OzoneVolume volume; private OzoneBucket bucket; + private boolean originalFileSystemPathEnabled; - /** - * Create a MiniOzoneCluster for testing. - *

- * Ozone is made active by setting OZONE_ENABLED = true - * - * @throws IOException - */ @BeforeAll - public static void init() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10); - OMRequestTestUtils.configureFSOptimizedPaths(conf, true); - startCluster(conf); + void init() throws Exception { + cluster = cluster(); + ozClient = cluster.newClient(); + store = ozClient.getObjectStore(); eTagProvider = MessageDigest.getInstance(OzoneConsts.MD5_HASH); + OmConfig omConfig = cluster().getOzoneManager().getConfig(); + originalFileSystemPathEnabled = omConfig.isFileSystemPathEnabled(); + omConfig.setFileSystemPathEnabled(true); } - /** - * Close OzoneClient and shutdown MiniOzoneCluster. - */ @AfterAll - public static void shutdown() throws IOException { - shutdownCluster(); - } - - - /** - * Create a MiniOzoneCluster for testing. - * @param conf Configurations to start the cluster. - * @throws Exception - */ - static void startCluster(OzoneConfiguration conf) throws Exception { - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .build(); - cluster.waitForClusterToBeReady(); - ozClient = OzoneClientFactory.getRpcClient(conf); - store = ozClient.getObjectStore(); - } - - /** - * Close OzoneClient and shutdown MiniOzoneCluster. - */ - static void shutdownCluster() throws IOException { - if (ozClient != null) { - ozClient.close(); - } - - if (cluster != null) { - cluster.shutdown(); - } + void shutdown() { + IOUtils.closeQuietly(ozClient); + OmConfig omConfig = cluster().getOzoneManager().getConfig(); + omConfig.setFileSystemPathEnabled(originalFileSystemPathEnabled); } @BeforeEach diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java index 44fde35bc8a0..df932ed34fc3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.client.rpc.TestCloseContainerHandlingByClient; import org.apache.hadoop.ozone.client.rpc.TestContainerStateMachineStream; import org.apache.hadoop.ozone.client.rpc.TestDiscardPreallocatedBlocks; +import org.apache.hadoop.ozone.client.rpc.TestOzoneClientMultipartUploadWithFSO; import org.apache.hadoop.ozone.client.rpc.TestOzoneRpcClientWithKeyLatestVersion; import org.apache.hadoop.ozone.om.TestBucketLayoutWithOlderClient; import org.apache.hadoop.ozone.om.TestListKeys; @@ -73,6 +74,14 @@ public interface TestCase { MiniOzoneCluster cluster(); } + @Nested + class OzoneClientMultipartUploadWithFSO extends TestOzoneClientMultipartUploadWithFSO { + @Override + public MiniOzoneCluster cluster() { + return getCluster(); + } + } + @Nested class OzoneFSBucketLayout extends TestOzoneFSBucketLayout { @Override From bab7ab7647b3dfa185025ccc6aad8595c59e24c3 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 20 Mar 2025 21:41:18 +0100 Subject: [PATCH 5/5] HDDS-12465. Intermittent failure in TestOzoneFileSystemMetrics --- .../src/test/java/org/apache/ozone/test/ClusterForTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java index b9264b1c8c0c..c91bca1a7119 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; import java.time.Duration; import java.util.concurrent.TimeUnit; @@ -69,6 +70,7 @@ protected static OzoneConfiguration createBaseConfiguration() { conf.setBoolean("ozone.client.hbase.enhancements.allowed", true); conf.setBoolean(OZONE_FS_HSYNC_ENABLED, true); conf.setTimeDuration(OZONE_OM_LEASE_SOFT_LIMIT, 0, TimeUnit.SECONDS); + conf.setInt(OZONE_KEY_DELETING_LIMIT_PER_TASK, 0); return conf; }