From fb9415337a11ce17cd708b01b268bdf0b0f04c1b Mon Sep 17 00:00:00 2001 From: Li Cheng Date: Fri, 13 Dec 2019 00:35:26 +0800 Subject: [PATCH 1/2] HDDS-2035 Implement datanode level CLI to reveal pipeline relation. --- .../apache/hadoop/hdds/scm/cli/SCMCLI.java | 2 + .../scm/cli/datanode/DatanodeCommands.java | 52 ++++++++ .../scm/cli/datanode/ListInfoSubcommand.java | 117 ++++++++++++++++++ .../hdds/scm/cli/datanode/package-info.java | 22 ++++ 4 files changed, 193 insertions(+) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java index 8c0fb0370834..20a35a8c4e21 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands; +import org.apache.hadoop.hdds.scm.cli.datanode.DatanodeCommands; import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands; import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; import org.apache.hadoop.hdds.scm.client.ScmClient; @@ -59,6 +60,7 @@ SafeModeCommands.class, ContainerCommands.class, PipelineCommands.class, + DatanodeCommands.class, TopologySubcommand.class, ReplicationManagerCommands.class }, diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java new file mode 100644 index 000000000000..94763d356cb2 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeCommands.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.datanode; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.MissingSubcommandException; +import org.apache.hadoop.hdds.scm.cli.SCMCLI; +import picocli.CommandLine; + +import java.util.concurrent.Callable; + +/** + * Subcommand for datanode related operations. + */ +@CommandLine.Command( + name = "datanode", + description = "Datanode specific operations", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class, + subcommands = { + ListInfoSubcommand.class + }) +public class DatanodeCommands implements Callable { + + @CommandLine.ParentCommand + private SCMCLI parent; + + public SCMCLI getParent() { + return parent; + } + + @Override + public Void call() throws Exception { + throw new MissingSubcommandException( + this.parent.getCmd().getSubcommands().get("datanode")); + } +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java new file mode 100644 index 000000000000..979cd3932661 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.datanode; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import picocli.CommandLine; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +/** + * Handler of list datanodes info command. + */ +@CommandLine.Command( + name = "list", + description = "List info of datanodes", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class ListInfoSubcommand implements Callable { + + @CommandLine.ParentCommand + private DatanodeCommands parent; + + @CommandLine.Option(names = {"-ip", "--byIp"}, + description = "Show info by ip address.", + defaultValue = "", + required = false) + private String ipaddress; + + @CommandLine.Option(names = {"-id", "--byUuid"}, + description = "Show info by datanode UUID.", + defaultValue = "", + required = false) + private String uuid; + + private List pipelines; + + + @Override + public Void call() throws Exception { + try (ScmClient scmClient = parent.getParent().createScmClient()) { + pipelines = scmClient.listPipelines(); + if (isNullOrEmpty(ipaddress) && isNullOrEmpty(uuid)) { + getAllNodes(scmClient).stream().forEach(p -> printDatanodeInfo(p)); + } else { + getAllNodes(scmClient).stream().filter( + p -> ((isNullOrEmpty(ipaddress) || + (p.getIpAddress().compareToIgnoreCase(ipaddress) == 0)) + && (isNullOrEmpty(uuid) || + (p.getUuid().equals(uuid))))) + .forEach(p -> printDatanodeInfo(p)); + } + return null; + } + } + + private List getAllNodes(ScmClient scmClient) + throws IOException { + List nodes = scmClient.queryNode( + HddsProtos.NodeState.HEALTHY, HddsProtos.QueryScope.CLUSTER, ""); + List datanodes = new ArrayList<>(nodes.size()); + nodes.stream().forEach( + p -> datanodes.add(DatanodeDetails.getFromProtoBuf(p.getNodeID()))); + return datanodes; + } + + private void printDatanodeInfo(DatanodeDetails datanode) { + if (pipelines.isEmpty()) { + System.out.println("Datanode: " + datanode.getUuid().toString() + + " (" + datanode.getIpAddress() + "/" + + datanode.getHostName() + "). \n No pipeline created."); + return; + } + List relatedPipelines = pipelines.stream().filter( + p -> p.getNodes().contains(datanode)).collect(Collectors.toList()); + StringBuilder pipelineList = new StringBuilder() + .append("Related pipelines:\n"); + relatedPipelines.stream().forEach( + p -> pipelineList.append(p.getId().getId().toString()) + .append("/").append(p.getFactor().toString()).append("/") + .append(p.getType().toString()).append("/") + .append(p.getPipelineState().toString()).append("/") + .append(datanode.getUuid().equals(p.getLeaderId()) ? + "Leader" : "Follower") + .append(System.getProperty("line.separator"))); + System.out.println("Datanode: " + datanode.getUuid().toString() + + " (" + datanode.getIpAddress() + "/" + + datanode.getHostName() + "/" + relatedPipelines.size() + + " pipelines). \n" + pipelineList); + } + + protected static boolean isNullOrEmpty(String str) { + return ((str == null) || str.trim().isEmpty()); + } +} \ No newline at end of file diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java new file mode 100644 index 000000000000..f4c45cfa0e3e --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Contains all of the datanode related scm commands. + */ +package org.apache.hadoop.hdds.scm.cli.datanode; \ No newline at end of file From febf91f027e57e11a78f2c6bc2ae860835a65cbe Mon Sep 17 00:00:00 2001 From: Li Cheng Date: Fri, 13 Dec 2019 22:43:52 +0800 Subject: [PATCH 2/2] Update filter syntax. --- .../scm/cli/datanode/ListInfoSubcommand.java | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 979cd3932661..dcd8402fef47 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -25,10 +25,10 @@ import picocli.CommandLine; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Handler of list datanodes info command. @@ -43,13 +43,13 @@ public class ListInfoSubcommand implements Callable { @CommandLine.ParentCommand private DatanodeCommands parent; - @CommandLine.Option(names = {"-ip", "--byIp"}, + @CommandLine.Option(names = {"--ip"}, description = "Show info by ip address.", defaultValue = "", required = false) private String ipaddress; - @CommandLine.Option(names = {"-id", "--byUuid"}, + @CommandLine.Option(names = {"--id"}, description = "Show info by datanode UUID.", defaultValue = "", required = false) @@ -65,12 +65,15 @@ public Void call() throws Exception { if (isNullOrEmpty(ipaddress) && isNullOrEmpty(uuid)) { getAllNodes(scmClient).stream().forEach(p -> printDatanodeInfo(p)); } else { - getAllNodes(scmClient).stream().filter( - p -> ((isNullOrEmpty(ipaddress) || - (p.getIpAddress().compareToIgnoreCase(ipaddress) == 0)) - && (isNullOrEmpty(uuid) || - (p.getUuid().equals(uuid))))) - .forEach(p -> printDatanodeInfo(p)); + Stream allNodes = getAllNodes(scmClient).stream(); + if (!isNullOrEmpty(ipaddress)) { + allNodes = allNodes.filter(p -> p.getIpAddress() + .compareToIgnoreCase(ipaddress) == 0); + } + if (!isNullOrEmpty(uuid)) { + allNodes = allNodes.filter(p -> p.getUuid().toString().equals(uuid)); + } + allNodes.forEach(p -> printDatanodeInfo(p)); } return null; } @@ -80,35 +83,39 @@ private List getAllNodes(ScmClient scmClient) throws IOException { List nodes = scmClient.queryNode( HddsProtos.NodeState.HEALTHY, HddsProtos.QueryScope.CLUSTER, ""); - List datanodes = new ArrayList<>(nodes.size()); - nodes.stream().forEach( - p -> datanodes.add(DatanodeDetails.getFromProtoBuf(p.getNodeID()))); - return datanodes; + + return nodes.stream() + .map(p -> DatanodeDetails.getFromProtoBuf(p.getNodeID())) + .collect(Collectors.toList()); } private void printDatanodeInfo(DatanodeDetails datanode) { - if (pipelines.isEmpty()) { - System.out.println("Datanode: " + datanode.getUuid().toString() + - " (" + datanode.getIpAddress() + "/" - + datanode.getHostName() + "). \n No pipeline created."); - return; + StringBuilder pipelineListInfo = new StringBuilder(); + int relatedPipelineNum = 0; + if (!pipelines.isEmpty()) { + List relatedPipelines = pipelines.stream().filter( + p -> p.getNodes().contains(datanode)).collect(Collectors.toList()); + if (relatedPipelines.isEmpty()) { + pipelineListInfo.append("No related pipelines" + + " or the node is not in Healthy state."); + } else { + relatedPipelineNum = relatedPipelines.size(); + relatedPipelines.stream().forEach( + p -> pipelineListInfo.append(p.getId().getId().toString()) + .append("/").append(p.getFactor().toString()).append("/") + .append(p.getType().toString()).append("/") + .append(p.getPipelineState().toString()).append("/") + .append(datanode.getUuid().equals(p.getLeaderId()) ? + "Leader" : "Follower") + .append(System.getProperty("line.separator"))); + } + } else { + pipelineListInfo.append("No pipelines in cluster."); } - List relatedPipelines = pipelines.stream().filter( - p -> p.getNodes().contains(datanode)).collect(Collectors.toList()); - StringBuilder pipelineList = new StringBuilder() - .append("Related pipelines:\n"); - relatedPipelines.stream().forEach( - p -> pipelineList.append(p.getId().getId().toString()) - .append("/").append(p.getFactor().toString()).append("/") - .append(p.getType().toString()).append("/") - .append(p.getPipelineState().toString()).append("/") - .append(datanode.getUuid().equals(p.getLeaderId()) ? - "Leader" : "Follower") - .append(System.getProperty("line.separator"))); System.out.println("Datanode: " + datanode.getUuid().toString() + " (" + datanode.getIpAddress() + "/" - + datanode.getHostName() + "/" + relatedPipelines.size() + - " pipelines). \n" + pipelineList); + + datanode.getHostName() + "/" + relatedPipelineNum + + " pipelines) \n" + "Related pipelines: \n" + pipelineListInfo); } protected static boolean isNullOrEmpty(String str) {