From 9e107c0b84ca58efe093267a17a509ef384ca637 Mon Sep 17 00:00:00 2001 From: deveshsingh Date: Fri, 5 Jan 2024 10:39:51 +0530 Subject: [PATCH 1/2] HDDS-8492. Intermittent timeout in TestStorageContainerManager#testBlockDeletionTransactions. --- .../scm/TestStorageContainerManager.java | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java index bc17c0d88199..0359cb276403 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.hdds.ratis.RatisHelper; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -78,6 +79,7 @@ import org.apache.hadoop.ozone.OzoneTestUtils; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.HddsDatanodeService; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; import org.apache.hadoop.ozone.container.common.states.endpoint.HeartbeatEndpointTask; @@ -145,6 +147,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL; import static org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils.setInternalState; import static org.apache.hadoop.hdds.scm.HddsTestUtils.mockRemoteUser; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.junit.Assert.fail; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; @@ -279,26 +282,37 @@ private void verifyPermissionDeniedException(Exception e, String userName) { public void testBlockDeletionTransactions() throws Exception { int numKeys = 5; OzoneConfiguration conf = new OzoneConfiguration(); - conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 100, + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 100, + DatanodeConfiguration datanodeConfiguration = conf.getObject( + DatanodeConfiguration.class); + datanodeConfiguration.setBlockDeletionInterval(Duration.ofMillis(100)); + conf.setFromObject(datanodeConfiguration); + ScmConfig scmConfig = conf.getObject(ScmConfig.class); + scmConfig.setBlockDeletionInterval(Duration.ofMillis(100)); + conf.setFromObject(scmConfig); + + conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY + + ".client.request.write.timeout", 30, TimeUnit.SECONDS); + conf.setTimeDuration(RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY + + ".client.request.watch.timeout", 30, TimeUnit.SECONDS); + conf.setInt("hdds.datanode.block.delete.threads.max", 5); + conf.setInt("hdds.datanode.block.delete.queue.limit", 32); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 50, + TimeUnit.MILLISECONDS); + conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS); - conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, - 3000, + conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS); conf.setInt(ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 5); - conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL, - 1, TimeUnit.SECONDS); - ScmConfig scmConfig = conf.getObject(ScmConfig.class); - scmConfig.setBlockDeletionInterval(Duration.ofSeconds(1)); - conf.setFromObject(scmConfig); + // Reset container provision size, otherwise only one container // is created by default. conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, numKeys); MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) - .setHbInterval(100) + .setHbInterval(50) .build(); cluster.waitForClusterToBeReady(); @@ -317,7 +331,7 @@ public void testBlockDeletionTransactions() throws Exception { OzoneTestUtils.closeContainers(keyInfo.getKeyLocationVersions(), cluster.getStorageContainerManager()); } - + Thread.sleep(3000); Map> containerBlocks = createDeleteTXLog( cluster.getStorageContainerManager(), delLog, keyLocations, helper); @@ -340,7 +354,7 @@ public void testBlockDeletionTransactions() throws Exception { } catch (IOException e) { return false; } - }, 1000, 10000); + }, 1000, 22000); Assert.assertTrue(helper.verifyBlocksWithTxnTable(containerBlocks)); // Continue the work, add some TXs that with known container names, // but unknown block IDs. @@ -462,7 +476,7 @@ public void testBlockDeletingThrottling() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 5); - conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL, + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); ScmConfig scmConfig = conf.getObject(ScmConfig.class); scmConfig.setBlockDeletionInterval(Duration.ofMillis(100)); @@ -847,7 +861,7 @@ public void testCloseContainerCommandOnRestart() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 5); - conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL, + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, numKeys); From 8e12db0ca34518b58f88b34fc32d49a731b7bd14 Mon Sep 17 00:00:00 2001 From: deveshsingh Date: Tue, 16 Jan 2024 15:40:27 +0530 Subject: [PATCH 2/2] HDDS-8492. Fixed review comments. --- .../org/apache/hadoop/ozone/scm/TestStorageContainerManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java index 239abae142a5..da36c17fff0d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManager.java @@ -324,7 +324,6 @@ public void testBlockDeletionTransactions() throws Exception { OzoneTestUtils.closeContainers(keyInfo.getKeyLocationVersions(), cluster.getStorageContainerManager()); } - Thread.sleep(3000); Map> containerBlocks = createDeleteTXLog( cluster.getStorageContainerManager(), delLog, keyLocations, helper);