Skip to content

HDDS-10514. Recon - Provide DN decommissioning detailed status and info inline with current CLI command output. #6376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sumitagrawl merged 17 commits into from
May 19, 2024
Merged

HDDS-10514. Recon - Provide DN decommissioning detailed status and info inline with current CLI command output. #6376

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
523cc1d
HDDS-10514. Recon - Provide DN decommissioning detailed status and in…
Mar 13, 2024
525ef35
HDDS-10514. Reverting this file change.
Mar 13, 2024
f1df560
HDDS-10514. Added API for UUID argument.
Mar 14, 2024
c229db3
HDDS-10514. Fixed findbugs.
Mar 14, 2024
eb57e7a
HDDS-10514. Added test cases for endPoint APIs.
Mar 15, 2024
c29ac0b
HDDS-9537. Intermittent failure in TestPipelineManagerMXBean.
Mar 16, 2024
fe6ae72
Revert "HDDS-9537. Intermittent failure in TestPipelineManagerMXBean."
Mar 16, 2024
378ed7a
HDDS-10514. Fixed JSON formatting error and test cases.
Mar 18, 2024
0f76ecc
HDDS-10514. Fixed findbugs.
Mar 18, 2024
8e90c06
HDDS-10514. Fixed test failed cases.
Mar 18, 2024
b1c22b8
HDDS-10514. Fixed spelling mistakes in javadoc comments.
Mar 20, 2024
0869a88
HDDS-10514. Fixed review comments.
Apr 4, 2024
ed958bd
HDDS-10514. Changed the approach to get decommission status for DNs.
May 15, 2024
0ab6ae8
HDDS-10514. Added test cases.
May 15, 2024
6513cc3
HDDS-10514. Handled review comments.
May 15, 2024
aa1b62b
HDDS-10514. Handled review comments.
May 15, 2024
b4bc438
HDDS-10514. Removed unused constant.
May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@

package org.apache.hadoop.hdds;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.protobuf.ServiceException;

import jakarta.annotation.Nonnull;
Expand All @@ -30,8 +35,11 @@
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.file.Path;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand All @@ -40,6 +48,8 @@
import java.util.OptionalInt;
import java.util.TreeMap;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.hadoop.conf.ConfigRedactor;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
Expand All @@ -49,6 +59,7 @@
import org.apache.hadoop.hdds.conf.ConfigurationException;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProtoOrBuilder;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
Expand Down Expand Up @@ -870,4 +881,68 @@ public static HddsProtos.UUID toProtobuf(UUID uuid) {
? Thread.currentThread().getStackTrace()
: null;
}



public static List<HddsProtos.Node> getDecommissioningNodesList(Stream<HddsProtos.Node> allNodes,
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
String uuid,
String ipAddress) {
List<HddsProtos.Node> decommissioningNodes;
if (!Strings.isNullOrEmpty(uuid)) {
decommissioningNodes = allNodes.filter(p -> p.getNodeID().getUuid()
.equals(uuid)).collect(Collectors.toList());
} else if (!Strings.isNullOrEmpty(ipAddress)) {
decommissioningNodes = allNodes.filter(p -> p.getNodeID().getIpAddress()
.compareToIgnoreCase(ipAddress) == 0).collect(Collectors.toList());
} else {
decommissioningNodes = allNodes.collect(Collectors.toList());
}
return decommissioningNodes;
}

public static JsonNode getBeansJsonNode(String metricsJson) throws IOException {
JsonNode jsonNode;
ObjectMapper objectMapper = new ObjectMapper();
JsonFactory factory = objectMapper.getFactory();
JsonParser parser = factory.createParser(metricsJson);
jsonNode = (JsonNode) objectMapper.readTree(parser).get("beans").get(0);
return jsonNode;
}

public static int getNumDecomNodes(JsonNode jsonNode) {
int numDecomNodes;
JsonNode totalDecom = jsonNode.get("DecommissioningMaintenanceNodesTotal");
numDecomNodes = (totalDecom == null ? -1 : Integer.parseInt(totalDecom.toString()));
return numDecomNodes;
}

@Nullable
public static Map<String, Object> getCountsMap(DatanodeDetails datanode, JsonNode counts, int numDecomNodes,
Map<String, Object> countsMap, String errMsg)
throws IOException {
for (int i = 1; i <= numDecomNodes; i++) {
if (datanode.getHostName().equals(counts.get("tag.datanode." + i).asText())) {
JsonNode pipelinesDN = counts.get("PipelinesWaitingToCloseDN." + i);
JsonNode underReplicatedDN = counts.get("UnderReplicatedDN." + i);
JsonNode unclosedDN = counts.get("UnclosedContainersDN." + i);
JsonNode startTimeDN = counts.get("StartTimeDN." + i);
if (pipelinesDN == null || underReplicatedDN == null || unclosedDN == null || startTimeDN == null) {
throw new IOException(errMsg);
}

int pipelines = Integer.parseInt(pipelinesDN.toString());
double underReplicated = Double.parseDouble(underReplicatedDN.toString());
double unclosed = Double.parseDouble(unclosedDN.toString());
long startTime = Long.parseLong(startTimeDN.toString());
Date date = new Date(startTime);
DateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss z");
countsMap.put("decommissionStartTime", formatter.format(date));
countsMap.put("numOfUnclosedPipelines", pipelines);
countsMap.put("numOfUnderReplicatedContainers", underReplicated);
countsMap.put("numOfUnclosedContainers", unclosed);
return countsMap;
}
}
return null;
}
}
47 changes: 7 additions & 40 deletions ...s/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@
*/
package org.apache.hadoop.hdds.scm.cli.datanode;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
Expand All @@ -32,11 +30,8 @@
import picocli.CommandLine;

import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -72,26 +67,21 @@ public class DecommissionStatusSubCommand extends ScmSubcommand {

@Override
public void execute(ScmClient scmClient) throws IOException {
List<HddsProtos.Node> decommissioningNodes;
Stream<HddsProtos.Node> allNodes = scmClient.queryNode(DECOMMISSIONING,
null, HddsProtos.QueryScope.CLUSTER, "").stream();
List<HddsProtos.Node> decommissioningNodes = HddsUtils.getDecommissioningNodesList(allNodes, uuid, ipAddress);
if (!Strings.isNullOrEmpty(uuid)) {
decommissioningNodes = allNodes.filter(p -> p.getNodeID().getUuid()
.equals(uuid)).collect(Collectors.toList());
if (decommissioningNodes.isEmpty()) {
System.err.println("Datanode: " + uuid + " is not in DECOMMISSIONING");
return;
}
} else if (!Strings.isNullOrEmpty(ipAddress)) {
decommissioningNodes = allNodes.filter(p -> p.getNodeID().getIpAddress()
.compareToIgnoreCase(ipAddress) == 0).collect(Collectors.toList());
if (decommissioningNodes.isEmpty()) {
System.err.println("Datanode: " + ipAddress + " is not in " +
"DECOMMISSIONING");
return;
}
} else {
decommissioningNodes = allNodes.collect(Collectors.toList());
if (!json) {
System.out.println("\nDecommission Status: DECOMMISSIONING - " +
decommissioningNodes.size() + " node(s)");
Expand All @@ -102,12 +92,8 @@ public void execute(ScmClient scmClient) throws IOException {
int numDecomNodes = -1;
JsonNode jsonNode = null;
if (metricsJson != null) {
ObjectMapper objectMapper = new ObjectMapper();
JsonFactory factory = objectMapper.getFactory();
JsonParser parser = factory.createParser(metricsJson);
jsonNode = (JsonNode) objectMapper.readTree(parser).get("beans").get(0);
JsonNode totalDecom = jsonNode.get("DecommissioningMaintenanceNodesTotal");
numDecomNodes = (totalDecom == null ? -1 : Integer.parseInt(totalDecom.toString()));
jsonNode = HddsUtils.getBeansJsonNode(metricsJson);
numDecomNodes = HddsUtils.getNumDecomNodes(jsonNode);
}

if (json) {
Expand Down Expand Up @@ -164,28 +150,9 @@ private Map<String, Object> getCounts(DatanodeDetails datanode, JsonNode counts,
Map<String, Object> countsMap = new LinkedHashMap<>();
String errMsg = getErrorMessage() + datanode.getHostName();
try {
for (int i = 1; i <= numDecomNodes; i++) {
if (datanode.getHostName().equals(counts.get("tag.datanode." + i).asText())) {
JsonNode pipelinesDN = counts.get("PipelinesWaitingToCloseDN." + i);
JsonNode underReplicatedDN = counts.get("UnderReplicatedDN." + i);
JsonNode unclosedDN = counts.get("UnclosedContainersDN." + i);
JsonNode startTimeDN = counts.get("StartTimeDN." + i);
if (pipelinesDN == null || underReplicatedDN == null || unclosedDN == null || startTimeDN == null) {
throw new IOException(errMsg);
}

int pipelines = Integer.parseInt(pipelinesDN.toString());
double underReplicated = Double.parseDouble(underReplicatedDN.toString());
double unclosed = Double.parseDouble(unclosedDN.toString());
long startTime = Long.parseLong(startTimeDN.toString());
Date date = new Date(startTime);
DateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss z");
countsMap.put("decommissionStartTime", formatter.format(date));
countsMap.put("numOfUnclosedPipelines", pipelines);
countsMap.put("numOfUnderReplicatedContainers", underReplicated);
countsMap.put("numOfUnclosedContainers", unclosed);
return countsMap;
}
countsMap = HddsUtils.getCountsMap(datanode, counts, numDecomNodes, countsMap, errMsg);
if (countsMap != null) {
return countsMap;
}
System.err.println(errMsg);
} catch (IOException e) {
Expand Down
4 changes: 4 additions & 0 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,10 @@ public final class ReconServerConfigKeys {

public static final int
OZONE_RECON_SCM_CLIENT_FAILOVER_MAX_RETRY_DEFAULT = 3;

public static final long
OZONE_RECON_COMMAND_PROCESS_TIME_OUT_DEFAULT = 60L;
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated

/**
* Private constructor for utility class.
*/
Expand Down
Loading