Skip to content

HDDS-9738. Display startTime, pipeline and container counts for decommissioning datanode #6083

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Tejaskriya wants to merge 4 commits into from
Closed

HDDS-9738. Display startTime, pipeline and container counts for decommissioning datanode #6083

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2d5c69a
HDDS-9738. Display startTime, pipeline and container counts for decom…
Jan 24, 2024
09ee091
Findbugs fixes
Jan 25, 2024
062be2e
Findbugs fix
Jan 30, 2024
64fe727
Fix url
Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ...op-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@
*/
package org.apache.hadoop.hdds.scm.cli.datanode;

import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.OzoneAdmin;
import org.apache.hadoop.hdds.cli.SubcommandWithParent;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.kohsuke.MetaInfServices;
import picocli.CommandLine;
import picocli.CommandLine.Model.CommandSpec;
Expand All @@ -46,6 +48,9 @@
@MetaInfServices(SubcommandWithParent.class)
public class DatanodeCommands implements Callable<Void>, SubcommandWithParent {

@CommandLine.ParentCommand
private OzoneAdmin parent;

@Spec
private CommandSpec spec;

Expand All @@ -55,6 +60,15 @@ public Void call() throws Exception {
return null;
}

public OzoneAdmin getParent() {
return parent;
}

@VisibleForTesting
public void setParent(OzoneConfiguration conf) {
parent = new OzoneAdmin(conf);
}

@Override
public Class<?> getParentType() {
return OzoneAdmin.class;
Expand Down
130 changes: 130 additions & 0 deletions ...s/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,49 @@
*/
package org.apache.hadoop.hdds.scm.cli.datanode;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.server.http.HttpConfig;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import picocli.CommandLine;

import javax.net.ssl.HttpsURLConnection;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static java.net.HttpURLConnection.HTTP_CREATED;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTPS_BIND_PORT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HTTP_BIND_PORT_DEFAULT;
import static org.apache.hadoop.hdds.server.http.HttpConfig.getHttpPolicy;
import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTPS_SCHEME;
import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTP_SCHEME;

/**
* Handler to print decommissioning nodes status.
Expand All @@ -45,6 +72,9 @@

public class DecommissionStatusSubCommand extends ScmSubcommand {

@CommandLine.ParentCommand
private StatusSubCommand parent;

@CommandLine.Option(names = { "--id" },
description = "Show info by datanode UUID",
defaultValue = "")
Expand Down Expand Up @@ -81,10 +111,19 @@ public void execute(ScmClient scmClient) throws IOException {
decommissioningNodes.size() + " node(s)");
}

Map<String, Object> counts = getCounts();
int numDecomNodes;
Double num = (Double) counts.get("DecommissioningMaintenanceNodesTotal");
if (num == null) {
numDecomNodes = -1;
} else {
numDecomNodes = num.intValue();
}
for (HddsProtos.Node node : decommissioningNodes) {
DatanodeDetails datanode = DatanodeDetails.getFromProtoBuf(
node.getNodeID());
printDetails(datanode);
printCounts(datanode, counts, numDecomNodes);
Map<String, List<ContainerID>> containers = scmClient.getContainersOnDecomNode(datanode);
System.out.println(containers);
}
Expand All @@ -94,4 +133,95 @@ private void printDetails(DatanodeDetails datanode) {
" (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
+ "/" + datanode.getHostName() + ")");
}
private void printCounts(DatanodeDetails datanode, Map<String, Object> counts, int numDecomNodes) {
try {
for (int i = 1; i <= numDecomNodes; i++) {
if (datanode.getHostName().equals(counts.get("tag.datanode." + i))) {
int pipelines = ((Double)counts.get("PipelinesWaitingToCloseDN." + i)).intValue();
int underReplicated = ((Double)counts.get("UnderReplicatedDN." + i)).intValue();
int unclosed = ((Double)counts.get("UnclosedContainersDN." + i)).intValue();
long startTime = ((Double)counts.get("StartTimeDN." + i)).longValue();
System.out.print("Decommission started at : ");
Date date = new Date(startTime);
DateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss z");
System.out.println(formatter.format(date));
System.out.println("No. of Pipelines: " + pipelines);
System.out.println("No. of UnderReplicated containers: " + underReplicated);
System.out.println("No. of Unclosed Containers: " + unclosed);
return;
}
}
System.err.println("Error getting pipeline and container counts for " + datanode.getHostName());
} catch (NullPointerException ex) {
System.err.println("Error getting pipeline and container counts for " + datanode.getHostName());
}
}

private Map<String, Object> getCounts() {
Map<String, Object> finalResult = new HashMap<>();
try {
StringBuffer url = new StringBuffer();
final OzoneConfiguration ozoneConf = parent
.getParent()
.getParent()
.getOzoneConf();
final String protocol;
final URLConnectionFactory connectionFactory = URLConnectionFactory.newDefaultURLConnectionFactory(ozoneConf);
final HttpConfig.Policy webPolicy = getHttpPolicy(ozoneConf);
String host;
InputStream inputStream;
int errorCode;

if (webPolicy.isHttpsEnabled()) {
Copy link
Contributor

@sodonnel sodonnel Jan 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good idea, to pull the metrics rather than having a special command for getting these details.

Does this work if Kerberos is enabled and the SCM webUI has kerberos authentication enabled too?

Also, what about HA SCM? We need to get the metrics from the active SCM, not the standbys as they will not have the correct metrics.

Copy link
Contributor Author

@Tejaskriya Tejaskriya Feb 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To solve these issues, as you had suggested during our discussions, a better way would be to have something similar to JMXJsonServerlet from hadoop-common library in ozone which can return any filtered metrics through grpc calls. This way we avoid dealing with http calls issues like handling security and finding the scm leader to get the right metrics. We would still getting the metrics from MBeansServer and not adding any significant overhead in scm.
I have raised this PR with this new approach: #6185
Please do review it. Thank you!

protocol = HTTPS_SCHEME;
host = ozoneConf.get(OZONE_SCM_HTTPS_ADDRESS_KEY,
OZONE_SCM_HTTP_BIND_HOST_DEFAULT + ":" + OZONE_SCM_HTTPS_BIND_PORT_DEFAULT);
url.append(protocol).append("://").append(host).append("/jmx")
.append("?qry=Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics");

HttpsURLConnection httpsURLConnection = (HttpsURLConnection) connectionFactory
.openConnection(new URL(url.toString()));
httpsURLConnection.connect();
errorCode = httpsURLConnection.getResponseCode();
inputStream = httpsURLConnection.getInputStream();
} else {
protocol = HTTP_SCHEME;
host = ozoneConf.get(OZONE_SCM_HTTP_ADDRESS_KEY,
OZONE_SCM_HTTP_BIND_HOST_DEFAULT + ":" + OZONE_SCM_HTTP_BIND_PORT_DEFAULT);
url.append(protocol + "://" + host).append("/jmx")
.append("?qry=Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics");

HttpURLConnection httpURLConnection = (HttpURLConnection) connectionFactory
.openConnection(new URL(url.toString()));
httpURLConnection.connect();
errorCode = httpURLConnection.getResponseCode();
inputStream = httpURLConnection.getInputStream();
}

if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
String response = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
HashMap<String, ArrayList<Map>> result = new Gson().fromJson(response, HashMap.class);
finalResult = result.get("beans").get(0);
return finalResult;
} else {
throw new IOException("Unable to retrieve pipeline and container counts.");
}
} catch (MalformedURLException ex) {
System.err.println("Unable to retrieve pipeline and container counts.");
return finalResult;
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}

public StatusSubCommand getParent() {
return parent;
}

@VisibleForTesting
public void setParent(OzoneConfiguration conf) {
parent = new StatusSubCommand();
parent.setParent(conf);
}

}
15 changes: 15 additions & 0 deletions ...op-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/StatusSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
* limitations under the License.
*/

import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.cli.SubcommandWithParent;
Expand All @@ -40,6 +42,9 @@
@MetaInfServices(SubcommandWithParent.class)
public class StatusSubCommand implements Callable<Void>, SubcommandWithParent {

@CommandLine.ParentCommand
private DatanodeCommands parent;

@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;

Expand All @@ -49,6 +54,16 @@ public Void call() throws Exception {
return null;
}

public DatanodeCommands getParent() {
return parent;
}

@VisibleForTesting
public void setParent(OzoneConfiguration conf) {
parent = new DatanodeCommands();
parent.setParent(conf);
}

@Override
public Class<?> getParentType() {
return DatanodeCommands.class;
Expand Down
43 changes: 43 additions & 0 deletions ...c/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDecommissionStatusSubCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,19 @@
*/
package org.apache.hadoop.hdds.scm.cli.datanode;

import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -29,6 +38,8 @@
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -60,12 +71,44 @@ public class TestDecommissionStatusSubCommand {
private DecommissionStatusSubCommand cmd;
private List<HddsProtos.Node> nodes = getNodeDetails(2);
private Map<String, List<ContainerID>> containerOnDecom = getContainersOnDecomNodes();
private static HttpServer httpServer;
private OzoneConfiguration conf = new OzoneConfiguration();

@BeforeAll
public static void setupScmHttp() throws Exception {
httpServer = HttpServer.create(new InetSocketAddress(15000), 0);
httpServer.createContext("/jmx", new HttpHandler() {
public void handle(HttpExchange exchange) throws IOException {
byte[] response = ("{ \"beans\" : [ { " +
"\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " +
"\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 0, " +
"\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 0, " +
"\"ContainersUnderReplicatedTotal\" : 0, \"ContainersUnClosedTotal\" : 0, " +
"\"ContainersSufficientlyReplicatedTotal\" : 0 } ]}").getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.length);
exchange.getResponseBody().write(response);
exchange.close();
}
});
httpServer.start();
}
@AfterAll
public static void shutdownScmHttp() {
if (httpServer != null) {
httpServer.stop(0);
}
}

@BeforeEach
public void setup() throws UnsupportedEncodingException {
cmd = new DecommissionStatusSubCommand();
System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
HttpConfig.Policy policy = HttpConfig.Policy.HTTP_ONLY;
conf.set(OzoneConfigKeys.OZONE_HTTP_POLICY_KEY, policy.name());
conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "localhost:15000");
conf.set(ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_KEY, "localhost");
cmd.setParent(conf);
}

@AfterEach
Expand Down