Skip to content

HDDS-10436. datanode status decommission command should have json output option #6369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devmadhuu merged 8 commits into from
Mar 21, 2024
Merged

HDDS-10436. datanode status decommission command should have json output option #6369

Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5d160ac
HDDS-10436. datanode status decommission command should have json out…
sarvekshayr Mar 12, 2024
b1c2870
Fixed checkstyle errors
sarvekshayr Mar 12, 2024
3d2abbc
Update hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli…
sarvekshayr Mar 13, 2024
90291da
Added checked exception
sarvekshayr Mar 14, 2024
e61a489
Error message in a variable
sarvekshayr Mar 14, 2024
043fa1d
Fixed checkstyle errors - 2
sarvekshayr Mar 14, 2024
e6e9537
Error message in a variable - 2
sarvekshayr Mar 14, 2024
7c0244e
Improved json output
sarvekshayr Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 105 additions & 9 deletions ...s/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.server.JsonUtils;
import picocli.CommandLine;

import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -62,6 +65,11 @@ public class DecommissionStatusSubCommand extends ScmSubcommand {
defaultValue = "")
private String ipAddress;

@CommandLine.Option(names = { "--json" },
description = "Show output in json format",
defaultValue = "false")
private boolean json;

@Override
public void execute(ScmClient scmClient) throws IOException {
List<HddsProtos.Node> decommissioningNodes;
Expand All @@ -84,8 +92,10 @@ public void execute(ScmClient scmClient) throws IOException {
}
} else {
decommissioningNodes = allNodes.collect(Collectors.toList());
System.out.println("\nDecommission Status: DECOMMISSIONING - " +
decommissioningNodes.size() + " node(s)");
if (!json) {
System.out.println("\nDecommission Status: DECOMMISSIONING - " +
decommissioningNodes.size() + " node(s)");
}
}

String metricsJson = scmClient.getMetrics("Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics");
Expand All @@ -100,6 +110,22 @@ public void execute(ScmClient scmClient) throws IOException {
numDecomNodes = (totalDecom == null ? -1 : Integer.parseInt(totalDecom.toString()));
}

if (json) {
List<Map<String, Object>> decommissioningNodesDetails = new ArrayList<>();

for (HddsProtos.Node node : decommissioningNodes) {
DatanodeDetails datanode = DatanodeDetails.getFromProtoBuf(
node.getNodeID());
Map<String, Object> datanodeMap = new LinkedHashMap<>();
datanodeMap.put("datanodeDetails", getDatanodeDetails(datanode));
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
datanodeMap.put("metrics", getCounts(datanode, jsonNode, numDecomNodes));
datanodeMap.put("containers", getContainers(scmClient, datanode));
Comment thread
sarvekshayr marked this conversation as resolved.
decommissioningNodesDetails.add(datanodeMap);
}
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(decommissioningNodesDetails));
return;
}

for (HddsProtos.Node node : decommissioningNodes) {
DatanodeDetails datanode = DatanodeDetails.getFromProtoBuf(
node.getNodeID());
Expand All @@ -110,6 +136,16 @@ public void execute(ScmClient scmClient) throws IOException {
}
}

private String errorMessage = "Error getting pipeline and container metrics for ";

public String getErrorMessage() {
return errorMessage;
}

public void setErrorMessage(String errorMessage) {
this.errorMessage = errorMessage;
}

private void printDetails(DatanodeDetails datanode) {
System.out.println("\nDatanode: " + datanode.getUuid().toString() +
" (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
Expand All @@ -120,10 +156,18 @@ private void printCounts(DatanodeDetails datanode, JsonNode counts, int numDecom
try {
for (int i = 1; i <= numDecomNodes; i++) {
if (datanode.getHostName().equals(counts.get("tag.datanode." + i).asText())) {
int pipelines = Integer.parseInt(counts.get("PipelinesWaitingToCloseDN." + i).toString());
double underReplicated = Double.parseDouble(counts.get("UnderReplicatedDN." + i).toString());
double unclosed = Double.parseDouble(counts.get("UnclosedContainersDN." + i).toString());
long startTime = Long.parseLong(counts.get("StartTimeDN." + i).toString());
JsonNode pipelinesDN = counts.get("PipelinesWaitingToCloseDN." + i);
JsonNode underReplicatedDN = counts.get("UnderReplicatedDN." + i);
JsonNode unclosedDN = counts.get("UnclosedContainersDN." + i);
JsonNode startTimeDN = counts.get("StartTimeDN." + i);
if (pipelinesDN == null || underReplicatedDN == null || unclosedDN == null || startTimeDN == null) {
throw new IOException("Error getting pipeline and container metrics for " + datanode.getHostName());
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
}

int pipelines = Integer.parseInt(pipelinesDN.toString());
double underReplicated = Double.parseDouble(underReplicatedDN.toString());
double unclosed = Double.parseDouble(unclosedDN.toString());
long startTime = Long.parseLong(startTimeDN.toString());
System.out.print("Decommission Started At : ");
Date date = new Date(startTime);
DateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss z");
Expand All @@ -134,9 +178,61 @@ private void printCounts(DatanodeDetails datanode, JsonNode counts, int numDecom
return;
}
}
System.err.println("Error getting pipeline and container counts for " + datanode.getHostName());
} catch (NullPointerException ex) {
System.err.println("Error getting pipeline and container counts for " + datanode.getHostName());
System.err.println(getErrorMessage() + datanode.getHostName());
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
} catch (IOException e) {
System.err.println(getErrorMessage() + datanode.getHostName());
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
}
}

private Map<String, Object> getDatanodeDetails(DatanodeDetails datanode) {
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
Map<String, Object> detailsMap = new LinkedHashMap<>();
detailsMap.put("uuid", datanode.getUuid().toString());
detailsMap.put("networkLocation", datanode.getNetworkLocation());
detailsMap.put("ipAddress", datanode.getIpAddress());
detailsMap.put("hostname", datanode.getHostName());
return detailsMap;
}

private Map<String, Object> getCounts(DatanodeDetails datanode, JsonNode counts, int numDecomNodes) {
Comment thread
sarvekshayr marked this conversation as resolved.
Map<String, Object> countsMap = new LinkedHashMap<>();
try {
for (int i = 1; i <= numDecomNodes; i++) {
if (datanode.getHostName().equals(counts.get("tag.datanode." + i).asText())) {
JsonNode pipelinesDN = counts.get("PipelinesWaitingToCloseDN." + i);
JsonNode underReplicatedDN = counts.get("UnderReplicatedDN." + i);
JsonNode unclosedDN = counts.get("UnclosedContainersDN." + i);
JsonNode startTimeDN = counts.get("StartTimeDN." + i);
if (pipelinesDN == null || underReplicatedDN == null || unclosedDN == null || startTimeDN == null) {
throw new IOException("Error getting pipeline and container metrics for " + datanode.getHostName());
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
}

int pipelines = Integer.parseInt(pipelinesDN.toString());
double underReplicated = Double.parseDouble(underReplicatedDN.toString());
double unclosed = Double.parseDouble(unclosedDN.toString());
long startTime = Long.parseLong(startTimeDN.toString());
Date date = new Date(startTime);
DateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss z");
countsMap.put("decommissionStartTime", formatter.format(date));
countsMap.put("numOfUnclosedPipelines", pipelines);
countsMap.put("numOfUnderReplicatedContainers", underReplicated);
countsMap.put("numOfUnclosedContainers", unclosed);
return countsMap;
}
}
System.err.println(getErrorMessage() + datanode.getHostName());
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
} catch (IOException e) {
System.err.println(getErrorMessage() + datanode.getHostName());
Comment thread
sarvekshayr marked this conversation as resolved.
Outdated
}
return countsMap;
}

private Map<String, Object> getContainers(ScmClient scmClient, DatanodeDetails datanode) throws IOException {
Comment thread
sarvekshayr marked this conversation as resolved.
Map<String, List<ContainerID>> containers = scmClient.getContainersOnDecomNode(datanode);
return containers.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,
entry -> entry.getValue().stream().
map(ContainerID::toString).
collect(Collectors.toList())));
}
}