Skip to content

HDDS-12443. Intermittent failure in TestContainerBalancerSubCommand #7989

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nandakumar131 merged 7 commits into from
Mar 2, 2025
Merged

HDDS-12443. Intermittent failure in TestContainerBalancerSubCommand #7989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1487be6
replace assertTrue/False with assertThat
adoroszlai Feb 28, 2025
75892d5
use PrintStreamCapturer
adoroszlai Feb 28, 2025
4067045
reuse patterns
adoroszlai Feb 28, 2025
4276f6b
reuse AssertionBuilder
adoroszlai Feb 28, 2025
beff80b
deduplicate balancerConfigOutput
adoroszlai Feb 28, 2025
3a8efcc
HDDS-12443. Intermittent failure in TestContainerBalancerSubCommand
adoroszlai Feb 28, 2025
c074f5b
make constants private
adoroszlai Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions hadoop-hdds/tools/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-test-utils</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
5 changes: 4 additions & 1 deletion ...tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
Expand Down Expand Up @@ -69,7 +70,9 @@ public void execute(ScmClient scmClient) throws IOException {
System.out.println("ContainerBalancer is Running.");

if (verbose) {
System.out.printf("Started at: %s %s%n", dateTime.toLocalDate(), dateTime.toLocalTime());
System.out.printf("Started at: %s %s%n",
dateTime.toLocalDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
dateTime.toLocalTime().format(DateTimeFormatter.ISO_LOCAL_TIME));
Duration balancingDuration = Duration.between(startedAtInstant, OffsetDateTime.now());
System.out.printf("Balancing duration: %s%n%n", getPrettyDuration(balancingDuration));
System.out.println(getConfigurationPrettyString(balancerStatusInfo.getConfiguration()));
Expand Down
210 changes: 69 additions & 141 deletions ...rc/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,13 @@
package org.apache.hadoop.hdds.scm.cli.datanode;

import static org.apache.hadoop.ozone.OzoneConsts.GB;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto;
Expand All @@ -40,6 +34,8 @@
import org.apache.hadoop.hdds.scm.cli.ContainerBalancerStopSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -51,14 +47,44 @@
*/
class TestContainerBalancerSubCommand {

private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
private final PrintStream originalOut = System.out;
private final PrintStream originalErr = System.err;
private static final Pattern DURATION = Pattern.compile(
"^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
private static final Pattern FAILED_TO_START = Pattern.compile(
"^Failed\\sto\\sstart\\sContainer\\sBalancer.");
private static final Pattern IS_NOT_RUNNING = Pattern.compile(
"^ContainerBalancer\\sis\\sNot\\sRunning.");
private static final Pattern IS_RUNNING = Pattern.compile(
"^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
private static final Pattern STARTED_AT = Pattern.compile(
"^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$", Pattern.MULTILINE);
private static final Pattern STARTED_SUCCESSFULLY = Pattern.compile(
"^Container\\sBalancer\\sstarted\\ssuccessfully.");
private static final Pattern WAITING_TO_STOP = Pattern.compile(
"^Sending\\sstop\\scommand.\\sWaiting\\sfor\\sContainer\\sBalancer\\sto\\sstop...\\n" +
"Container\\sBalancer\\sstopped.");

private static final String BALANCER_CONFIG_OUTPUT = "Container Balancer Configuration values:\n" +
"Key Value\n" +
"Threshold 10.0\n" +
"Max Datanodes to Involve per Iteration(percent) 20\n" +
"Max Size to Move per Iteration 0GB\n" +
"Max Size Entering Target per Iteration 26GB\n" +
"Max Size Leaving Source per Iteration 26GB\n" +
"Number of Iterations 3\n" +
"Time Limit for Single Container's Movement 65min\n" +
"Time Limit for Single Container's Replication 50min\n" +
"Interval between each Iteration 0min\n" +
"Whether to Enable Network Topology false\n" +
"Whether to Trigger Refresh Datanode Usage Info false\n" +
"Container IDs to Exclude from Balancing None\n" +
"Datanodes Specified to be Balanced None\n" +
"Datanodes Excluded from Balancing None";

private ContainerBalancerStopSubcommand stopCmd;
private ContainerBalancerStartSubcommand startCmd;
private ContainerBalancerStatusSubcommand statusCmd;
private GenericTestUtils.PrintStreamCapturer out;
private GenericTestUtils.PrintStreamCapturer err;

private static ContainerBalancerStatusInfoResponseProto getContainerBalancerStatusInfoResponseProto(
ContainerBalancerConfiguration config) {
Expand Down Expand Up @@ -203,18 +229,17 @@ private static ContainerBalancerConfiguration getContainerBalancerConfiguration(
}

@BeforeEach
public void setup() throws UnsupportedEncodingException {
void setup() {
stopCmd = new ContainerBalancerStopSubcommand();
startCmd = new ContainerBalancerStartSubcommand();
statusCmd = new ContainerBalancerStatusSubcommand();
System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
out = GenericTestUtils.captureOut();
err = GenericTestUtils.captureErr();
}

@AfterEach
public void tearDown() {
System.setOut(originalOut);
System.setErr(originalErr);
void tearDown() {
IOUtils.closeQuietly(out, err);
}

@Test
Expand All @@ -230,30 +255,6 @@ void testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
//test status is running
when(scmClient.getContainerBalancerStatusInfo()).thenReturn(statusInfoResponseProto);
statusCmd.execute(scmClient);
Pattern p = Pattern.compile(
"^ContainerBalancer\\sis\\sRunning.");
String output = outContent.toString(DEFAULT_ENCODING);
Matcher m = p.matcher(output);
assertTrue(m.find());

String balancerConfigOutput =
"Container Balancer Configuration values:\n" +
"Key Value\n" +
"Threshold 10.0\n" +
"Max Datanodes to Involve per Iteration(percent) 20\n" +
"Max Size to Move per Iteration 0GB\n" +
"Max Size Entering Target per Iteration 26GB\n" +
"Max Size Leaving Source per Iteration 26GB\n" +
"Number of Iterations 3\n" +
"Time Limit for Single Container's Movement 65min\n" +
"Time Limit for Single Container's Replication 50min\n" +
"Interval between each Iteration 0min\n" +
"Whether to Enable Network Topology false\n" +
"Whether to Trigger Refresh Datanode Usage Info false\n" +
"Container IDs to Exclude from Balancing None\n" +
"Datanodes Specified to be Balanced None\n" +
"Datanodes Excluded from Balancing None";
assertFalse(output.contains(balancerConfigOutput));

String currentIterationOutput =
"Current iteration info:\n" +
Expand All @@ -273,9 +274,11 @@ void testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
assertFalse(output.contains(currentIterationOutput));

assertFalse(output.contains("Iteration history list:"));
assertThat(out.get()).containsPattern(IS_RUNNING)
.doesNotContain(BALANCER_CONFIG_OUTPUT)
.doesNotContain(currentIterationOutput)
.doesNotContain("Iteration history list:");
}

@Test
Expand All @@ -293,42 +296,7 @@ void testContainerBalancerStatusInfoSubcommandVerboseHistory()
CommandLine c = new CommandLine(statusCmd);
c.parseArgs("--verbose", "--history");
statusCmd.execute(scmClient);
String output = outContent.toString(DEFAULT_ENCODING);
Pattern p = Pattern.compile(
"^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
Matcher m = p.matcher(output);
assertTrue(m.find());

p = Pattern.compile(
"^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$", Pattern.MULTILINE);
m = p.matcher(output);
assertTrue(m.find());

p = Pattern.compile(
"^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
m = p.matcher(output);
assertTrue(m.find());

String balancerConfigOutput =
"Container Balancer Configuration values:\n" +
"Key Value\n" +
"Threshold 10.0\n" +
"Max Datanodes to Involve per Iteration(percent) 20\n" +
"Max Size to Move per Iteration 0GB\n" +
"Max Size Entering Target per Iteration 26GB\n" +
"Max Size Leaving Source per Iteration 26GB\n" +
"Number of Iterations 3\n" +
"Time Limit for Single Container's Movement 65min\n" +
"Time Limit for Single Container's Replication 50min\n" +
"Interval between each Iteration 0min\n" +
"Whether to Enable Network Topology false\n" +
"Whether to Trigger Refresh Datanode Usage Info false\n" +
"Container IDs to Exclude from Balancing None\n" +
"Datanodes Specified to be Balanced None\n" +
"Datanodes Excluded from Balancing None";
assertTrue(output.contains(balancerConfigOutput));

assertTrue(output.contains("Iteration history list:"));

String firstHistoryIterationOutput =
"Key Value\n" +
"Iteration number 3\n" +
Expand All @@ -346,7 +314,6 @@ void testContainerBalancerStatusInfoSubcommandVerboseHistory()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
assertTrue(output.contains(firstHistoryIterationOutput));

String secondHistoryIterationOutput =
"Key Value\n" +
Expand All @@ -365,7 +332,15 @@ void testContainerBalancerStatusInfoSubcommandVerboseHistory()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 15 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 15 GB";
assertTrue(output.contains(secondHistoryIterationOutput));

assertThat(out.get())
.containsPattern(IS_RUNNING)
.containsPattern(STARTED_AT)
.containsPattern(DURATION)
.contains(BALANCER_CONFIG_OUTPUT)
.contains("Iteration history list:")
.contains(firstHistoryIterationOutput)
.contains(secondHistoryIterationOutput);
}

@Test
Expand All @@ -383,40 +358,6 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
CommandLine c = new CommandLine(statusCmd);
c.parseArgs("--verbose");
statusCmd.execute(scmClient);
String output = outContent.toString(DEFAULT_ENCODING);
Pattern p = Pattern.compile(
"^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
Matcher m = p.matcher(output);
assertTrue(m.find());

p = Pattern.compile(
"^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$", Pattern.MULTILINE);
m = p.matcher(output);
assertTrue(m.find());

p = Pattern.compile(
"^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
m = p.matcher(output);
assertTrue(m.find());

String balancerConfigOutput =
"Container Balancer Configuration values:\n" +
"Key Value\n" +
"Threshold 10.0\n" +
"Max Datanodes to Involve per Iteration(percent) 20\n" +
"Max Size to Move per Iteration 0GB\n" +
"Max Size Entering Target per Iteration 26GB\n" +
"Max Size Leaving Source per Iteration 26GB\n" +
"Number of Iterations 3\n" +
"Time Limit for Single Container's Movement 65min\n" +
"Time Limit for Single Container's Replication 50min\n" +
"Interval between each Iteration 0min\n" +
"Whether to Enable Network Topology false\n" +
"Whether to Trigger Refresh Datanode Usage Info false\n" +
"Container IDs to Exclude from Balancing None\n" +
"Datanodes Specified to be Balanced None\n" +
"Datanodes Excluded from Balancing None";
assertTrue(output.contains(balancerConfigOutput));

String currentIterationOutput =
"Current iteration info:\n" +
Expand All @@ -436,9 +377,14 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
"Exited data from nodes \n" +
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
assertTrue(output.contains(currentIterationOutput));

assertFalse(output.contains("Iteration history list:"));
assertThat(out.get())
.containsPattern(IS_RUNNING)
.containsPattern(STARTED_AT)
.containsPattern(DURATION)
.contains(BALANCER_CONFIG_OUTPUT)
.contains(currentIterationOutput)
.doesNotContain("Iteration history list:");
}

@Test
Expand All @@ -453,10 +399,7 @@ void testContainerBalancerStatusInfoSubcommandRunningOnStoppedBalancer()
.build());

statusCmd.execute(scmClient);
Pattern p = Pattern.compile(
"^ContainerBalancer\\sis\\sNot\\sRunning.");
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
assertThat(out.get()).containsPattern(IS_NOT_RUNNING);
}

@Test
Expand All @@ -471,23 +414,15 @@ void testContainerBalancerStatusSubcommandNotRunning()

statusCmd.execute(scmClient);

Pattern p = Pattern.compile(
"^ContainerBalancer\\sis\\sNot\\sRunning.");
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
assertThat(out.get()).containsPattern(IS_NOT_RUNNING);
}

@Test
public void testContainerBalancerStopSubcommand() throws IOException {
ScmClient scmClient = mock(ScmClient.class);
stopCmd.execute(scmClient);

Pattern p = Pattern.compile("^Sending\\sstop\\scommand." +
"\\sWaiting\\sfor\\sContainer\\sBalancer\\sto\\sstop...\\n" +
"Container\\sBalancer\\sstopped.");

Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
assertThat(out.get()).containsPattern(WAITING_TO_STOP);
}

@Test
Expand All @@ -503,10 +438,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsNotRunning()
.build());
startCmd.execute(scmClient);

Pattern p = Pattern.compile("^Container\\sBalancer\\sstarted" +
"\\ssuccessfully.");
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
assertThat(out.get()).containsPattern(STARTED_SUCCESSFULLY);
}

@Test
Expand All @@ -522,11 +454,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsRunning()
.build());
startCmd.execute(scmClient);

Pattern p = Pattern.compile("^Failed\\sto\\sstart\\sContainer" +
"\\sBalancer.");

Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
assertTrue(m.find());
assertThat(out.get()).containsPattern(FAILED_TO_START);
}

}