From 8d861a2d8dd13caff5836db9b33e40d3ab67cdaf Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 7 Mar 2025 08:13:13 +0100 Subject: [PATCH 1/3] HDDS-12489. Intermittent timeout in TestSCMContainerManagerMetrics.testReportProcessingMetrics --- .../container/metrics/TestSCMContainerManagerMetrics.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java index 531a43eb4bba..0a7f433984a4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java @@ -143,8 +143,6 @@ public void testReportProcessingMetrics() throws Exception { assertThat(getLongCounter("NumContainerReportsProcessedSuccessful", metrics)) .isPositive(); - final long previous = getLongCounter("NumICRReportsProcessedSuccessful", metrics); - // Create key should create container on DN. client.getObjectStore().getClientProxy() .createVolume(volumeName); @@ -163,8 +161,7 @@ public void testReportProcessingMetrics() throws Exception { GenericTestUtils.waitFor(() -> { final MetricsRecordBuilder scmMetrics = getMetrics(SCMContainerManagerMetrics.class.getSimpleName()); - return getLongCounter("NumICRReportsProcessedSuccessful", - scmMetrics) >= previous + 1; + return getLongCounter("NumICRReportsProcessedSuccessful", scmMetrics) > 0; }, 100, 30_000); } } From 937bb62043a63d8ff8344885f42169e9730a6387 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 7 Mar 2025 08:14:00 +0100 Subject: [PATCH 2/3] simplify key creation --- .../TestSCMContainerManagerMetrics.java | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java index 0a7f433984a4..ef5bab8c0cd7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.container.metrics; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ozone.test.MetricsAsserts.getLongCounter; import static org.apache.ozone.test.MetricsAsserts.getMetrics; import static org.assertj.core.api.Assertions.assertThat; @@ -25,13 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; -import java.util.HashMap; -import java.util.UUID; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -41,8 +36,8 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterEach; @@ -134,29 +129,13 @@ public void testContainerOpsMetrics() throws Exception { @Test public void testReportProcessingMetrics() throws Exception { - String volumeName = "vol-" + UUID.randomUUID(); - String bucketName = "bucket1"; - String key = "key1"; - MetricsRecordBuilder metrics = getMetrics(SCMContainerManagerMetrics.class.getSimpleName()); assertThat(getLongCounter("NumContainerReportsProcessedSuccessful", metrics)) .isPositive(); // Create key should create container on DN. - client.getObjectStore().getClientProxy() - .createVolume(volumeName); - client.getObjectStore().getClientProxy() - .createBucket(volumeName, bucketName); - OzoneOutputStream ozoneOutputStream = client - .getObjectStore().getClientProxy().createKey(volumeName, bucketName, - key, 0, ReplicationType.RATIS, ReplicationFactor.ONE, - new HashMap<>()); - - String data = "file data"; - ozoneOutputStream.write(data.getBytes(UTF_8), 0, data.length()); - ozoneOutputStream.close(); - + TestDataUtil.createKeys(cluster(), 1); GenericTestUtils.waitFor(() -> { final MetricsRecordBuilder scmMetrics = From 50130c86348bab1b9a8df64f86cde2d6fc63982e Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 8 Mar 2025 03:01:36 +0100 Subject: [PATCH 3/3] closeAllContainers before creating key --- .../container/metrics/TestSCMContainerManagerMetrics.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java index ef5bab8c0cd7..1acb081c10ad 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/metrics/TestSCMContainerManagerMetrics.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneTestUtils; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.ozone.test.GenericTestUtils; @@ -134,13 +135,16 @@ public void testReportProcessingMetrics() throws Exception { assertThat(getLongCounter("NumContainerReportsProcessedSuccessful", metrics)) .isPositive(); + final long previous = getLongCounter("NumICRReportsProcessedSuccessful", metrics); + OzoneTestUtils.closeAllContainers(scm.getEventQueue(), scm); + // Create key should create container on DN. TestDataUtil.createKeys(cluster(), 1); GenericTestUtils.waitFor(() -> { final MetricsRecordBuilder scmMetrics = getMetrics(SCMContainerManagerMetrics.class.getSimpleName()); - return getLongCounter("NumICRReportsProcessedSuccessful", scmMetrics) > 0; + return getLongCounter("NumICRReportsProcessedSuccessful", scmMetrics) > previous; }, 100, 30_000); } }