diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java new file mode 100644 index 000000000000..4a925ffad5ff --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli.datanode; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + +/** + * Represents filtered Datanode information for json use. + */ +public class BasicDatanodeInfo { + @JsonInclude(JsonInclude.Include.NON_NULL) + private Long used = null; + @JsonInclude(JsonInclude.Include.NON_NULL) + private Long capacity = null; + @JsonInclude(JsonInclude.Include.NON_NULL) + private Double percentUsed = null; + private final DatanodeDetails dn; + private final HddsProtos.NodeOperationalState opState; + private final HddsProtos.NodeState healthState; + + public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState healthState) { + this.dn = dnDetails; + this.opState = opState; + this.healthState = healthState; + } + + public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState healthState, long used, long capacity, double percentUsed) { + this(dnDetails, opState, healthState); + this.used = used; + this.capacity = capacity; + this.percentUsed = percentUsed; + } + + @JsonProperty(index = 5) + public String getId() { + return dn.getUuidString(); + } + + @JsonProperty(index = 10) + public String getHostName() { + return dn.getHostName(); + } + + @JsonProperty(index = 15) + public String getIpAddress() { + return dn.getIpAddress(); + } + + @JsonProperty(index = 20) + public List getPorts() { + return dn.getPorts(); + } + + @JsonProperty(index = 25) + public long getSetupTime() { + return dn.getSetupTime(); + } + + @JsonProperty(index = 30) + public int getCurrentVersion() { + return dn.getCurrentVersion(); + } + + @JsonProperty(index = 35) + public HddsProtos.NodeOperationalState getOpState() { + return opState; + } + + @JsonProperty(index = 40) + public HddsProtos.NodeOperationalState getPersistedOpState() { + return dn.getPersistedOpState(); + } + + @JsonProperty(index = 45) + public long getPersistedOpStateExpiryEpochSec() { + return dn.getPersistedOpStateExpiryEpochSec(); + } + + @JsonProperty(index = 50) + public HddsProtos.NodeState getHealthState() { + return healthState; + } + + @JsonProperty(index = 55) + public boolean isDecommissioned() { + return dn.isDecommissioned(); + } + + @JsonProperty(index = 60) + public boolean isMaintenance() { + return dn.isMaintenance(); + } + + @JsonProperty(index = 65) + public int getLevel() { + return dn.getLevel(); + } + + @JsonProperty(index = 70) + public int getCost() { + return dn.getCost(); + } + + @JsonProperty(index = 75) + public int getNumOfLeaves() { + return dn.getNumOfLeaves(); + } + + @JsonProperty(index = 80) + public String getNetworkFullPath() { + return dn.getNetworkFullPath(); + } + + @JsonProperty(index = 85) + public String getNetworkLocation() { + return dn.getNetworkLocation(); + } + + @JsonProperty(index = 90) + public String getNetworkName() { + return dn.getNetworkName(); + } + + @JsonProperty(index = 95) + public Long getUsed() { + return used; + } + + @JsonProperty(index = 100) + public Long getCapacity() { + return capacity; + } + + @JsonProperty(index = 105) + public Double getPercentUsed() { + return percentUsed; + } + + @JsonIgnore + public DatanodeDetails getDatanodeDetails() { + return dn; + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index a30e91150e34..49fb032d6ce1 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -17,10 +17,10 @@ package org.apache.hadoop.hdds.scm.cli.datanode; -import com.fasterxml.jackson.annotation.JsonInclude; import com.google.common.base.Strings; import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -88,26 +88,23 @@ public void execute(ScmClient scmClient) throws IOException { pipelines = scmClient.listPipelines(); if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getNodeId())) { HddsProtos.Node node = scmClient.queryNode(UUID.fromString(exclusiveNodeOptions.getNodeId())); - DatanodeWithAttributes dwa = new DatanodeWithAttributes(DatanodeDetails - .getFromProtoBuf(node.getNodeID()), - node.getNodeOperationalStates(0), - node.getNodeStates(0)); - + BasicDatanodeInfo singleNodeInfo = new BasicDatanodeInfo(DatanodeDetails.getFromProtoBuf(node.getNodeID()), + node.getNodeOperationalStates(0), node.getNodeStates(0)); if (json) { - List singleList = Collections.singletonList(dwa); - System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(singleList)); + List dtoList = Collections.singletonList(singleNodeInfo); + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList)); } else { - printDatanodeInfo(dwa); + printDatanodeInfo(singleNodeInfo); } return; } - Stream allNodes = getAllNodes(scmClient).stream(); + Stream allNodes = getAllNodes(scmClient).stream(); if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getIp())) { - allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress() + allNodes = allNodes.filter(p -> p.getIpAddress() .compareToIgnoreCase(exclusiveNodeOptions.getIp()) == 0); } if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getHostname())) { - allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName() + allNodes = allNodes.filter(p -> p.getHostName() .compareToIgnoreCase(exclusiveNodeOptions.getHostname()) == 0); } if (!Strings.isNullOrEmpty(nodeOperationalState)) { @@ -124,42 +121,49 @@ public void execute(ScmClient scmClient) throws IOException { } if (json) { - List datanodeList = allNodes.collect( - Collectors.toList()); - System.out.println( - JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); + List datanodeList = allNodes.collect(Collectors.toList()); + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); } else { allNodes.forEach(this::printDatanodeInfo); } } - private List getAllNodes(ScmClient scmClient) + private List getAllNodes(ScmClient scmClient) throws IOException { // If sorting is requested if (exclusiveNodeOptions != null && (exclusiveNodeOptions.mostUsed || exclusiveNodeOptions.leastUsed)) { boolean sortByMostUsed = exclusiveNodeOptions.mostUsed; - List usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, - Integer.MAX_VALUE); + List usageInfos; + try { + usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, Integer.MAX_VALUE); + } catch (Exception e) { + System.err.println("Failed to get datanode usage info: " + e.getMessage()); + return Collections.emptyList(); + } return usageInfos.stream() .map(p -> { - String uuidStr = p.getNode().getUuid(); - UUID parsedUuid = UUID.fromString(uuidStr); - try { + String uuidStr = p.getNode().getUuid(); + UUID parsedUuid = UUID.fromString(uuidStr); HddsProtos.Node node = scmClient.queryNode(parsedUuid); long capacity = p.getCapacity(); long used = capacity - p.getRemaining(); double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0; - return new DatanodeWithAttributes( + return new BasicDatanodeInfo( DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeOperationalStates(0), node.getNodeStates(0), used, capacity, percentUsed); - } catch (IOException e) { + } catch (Exception e) { + String reason = "Could not process info for an unknown datanode"; + if (p != null && p.getNode() != null && !Strings.isNullOrEmpty(p.getNode().getUuid())) { + reason = "Could not process node info for " + p.getNode().getUuid(); + } + System.err.printf("Error: %s: %s%n", reason, e.getMessage()); return null; } }) @@ -170,17 +174,16 @@ private List getAllNodes(ScmClient scmClient) List nodes = scmClient.queryNode(null, null, HddsProtos.QueryScope.CLUSTER, ""); - return nodes.stream() - .map(p -> new DatanodeWithAttributes( + return nodes.stream().map(p -> new BasicDatanodeInfo( DatanodeDetails.getFromProtoBuf(p.getNodeID()), p.getNodeOperationalStates(0), p.getNodeStates(0))) - .sorted((o1, o2) -> o1.healthState.compareTo(o2.healthState)) + .sorted(Comparator.comparing(BasicDatanodeInfo::getHealthState)) .collect(Collectors.toList()); } - private void printDatanodeInfo(DatanodeWithAttributes dna) { + private void printDatanodeInfo(BasicDatanodeInfo dn) { StringBuilder pipelineListInfo = new StringBuilder(); - DatanodeDetails datanode = dna.getDatanodeDetails(); + DatanodeDetails datanode = dn.getDatanodeDetails(); int relatedPipelineNum = 0; if (!pipelines.isEmpty()) { List relatedPipelines = pipelines.stream().filter( @@ -206,72 +209,14 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) { " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress() + "/" + datanode.getHostName() + "/" + relatedPipelineNum + " pipelines)"); - System.out.println("Operational State: " + dna.getOpState()); - System.out.println("Health State: " + dna.getHealthState()); + System.out.println("Operational State: " + dn.getOpState()); + System.out.println("Health State: " + dn.getHealthState()); System.out.println("Related pipelines:\n" + pipelineListInfo); - if (dna.getUsed() != null && dna.getCapacity() != null && dna.getUsed() >= 0 && dna.getCapacity() > 0) { - System.out.println("Capacity: " + dna.getCapacity()); - System.out.println("Used: " + dna.getUsed()); - System.out.printf("Percentage Used : %.2f%%%n%n", dna.getPercentUsed()); - } - } - - private static class DatanodeWithAttributes { - private DatanodeDetails datanodeDetails; - private HddsProtos.NodeOperationalState operationalState; - private HddsProtos.NodeState healthState; - @JsonInclude(JsonInclude.Include.NON_NULL) - private Long used = null; - @JsonInclude(JsonInclude.Include.NON_NULL) - private Long capacity = null; - @JsonInclude(JsonInclude.Include.NON_NULL) - private Double percentUsed = null; - - DatanodeWithAttributes(DatanodeDetails dn, - HddsProtos.NodeOperationalState opState, - HddsProtos.NodeState healthState) { - this.datanodeDetails = dn; - this.operationalState = opState; - this.healthState = healthState; - } - - DatanodeWithAttributes(DatanodeDetails dn, - HddsProtos.NodeOperationalState opState, - HddsProtos.NodeState healthState, - long used, - long capacity, - double percentUsed) { - this.datanodeDetails = dn; - this.operationalState = opState; - this.healthState = healthState; - this.used = used; - this.capacity = capacity; - this.percentUsed = percentUsed; - } - - public DatanodeDetails getDatanodeDetails() { - return datanodeDetails; - } - - public HddsProtos.NodeOperationalState getOpState() { - return operationalState; - } - - public HddsProtos.NodeState getHealthState() { - return healthState; - } - - public Long getUsed() { - return used; - } - - public Long getCapacity() { - return capacity; - } - - public Double getPercentUsed() { - return percentUsed; + if (dn.getUsed() != null && dn.getCapacity() != null && dn.getUsed() >= 0 && dn.getCapacity() > 0) { + System.out.println("Capacity: " + dn.getCapacity()); + System.out.println("Used: " + dn.getUsed()); + System.out.printf("Percentage Used : %.2f%%%n%n", dn.getPercentUsed()); } } } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java index a63d0c925f42..fd6834068eef 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -207,9 +207,8 @@ public void testDataNodeByUuidOutput() assertEquals(1, root.size(), "Expected 1 node in JSON output"); JsonNode node = root.get(0); - assertTrue(node.has("datanodeDetails"), "Missing datanodeDetails"); String opState = node.get("opState").asText(); - String uuid = node.get("datanodeDetails").get("uuid").asText(); + String uuid = node.get("id").asText(); assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got: " + opState); assertEquals(nodes.get(0).getNodeID().getUuid(), uuid, @@ -365,8 +364,6 @@ private List getNodeDetails() { dnd.setIpAddress("1.2.3." + i + 1); dnd.setNetworkLocation("/default"); dnd.setNetworkName("host" + i); - dnd.addPorts(HddsProtos.Port.newBuilder() - .setName("ratis").setValue(5678).build()); dnd.setUuid(UUID.randomUUID().toString()); HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot index d4e7e7a2b48b..2795ae5ac4a6 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot @@ -99,9 +99,10 @@ Incomplete command List datanodes as JSON ${output} = Execute ozone admin datanode list --json | jq -r '.' - Should contain ${output} datanodeDetails + Should contain ${output} id Should contain ${output} healthState Should contain ${output} opState + Should contain ${output} persistedOpState Get usage info as JSON ${output} = Execute ozone admin datanode usageinfo -m --json | jq -r '.' diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 999aecb2fcf7..0436348239be 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -150,7 +150,7 @@ All container is closed Get Datanode Ozone Used Bytes Info [arguments] ${uuid} - ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].datanodeDetails.uuid'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add + ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add ${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value' [return] ${result}