HDDS-13101. Remove duplicate information in datanode list output

hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java

@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+/**
+ * Represents filtered Datanode information for json use.
+ */
+public class BasicDatanodeInfo {
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  private Long used = null;
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  private Long capacity = null;
+  @JsonInclude(JsonInclude.Include.NON_NULL)
+  private Double percentUsed = null;
+  private final DatanodeDetails dn;
+  private final HddsProtos.NodeOperationalState opState;
+  private final HddsProtos.NodeState healthState;
+
+  public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
+      HddsProtos.NodeState healthState) {
+    this.dn = dnDetails;
+    this.opState = opState;
+    this.healthState = healthState;
+  }
+
+  public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
+      HddsProtos.NodeState healthState, long used, long capacity, double percentUsed) {
+    this(dnDetails, opState, healthState);
+    this.used = used;
+    this.capacity = capacity;
+    this.percentUsed = percentUsed;
+  }
+
+  @JsonProperty(index = 5)
+  public String getId() {
+    return dn.getUuidString();
+  }
+
+  @JsonProperty(index = 10)
+  public String getHostName() {
+    return dn.getHostName();
+  }
+
+  @JsonProperty(index = 15)
+  public String getIpAddress() {
+    return dn.getIpAddress();
+  }
+
+  @JsonProperty(index = 20)
+  public List<DatanodeDetails.Port> getPorts() {
+    return dn.getPorts();
+  }
+
+  @JsonProperty(index = 25)
+  public long getSetupTime() {
+    return dn.getSetupTime();
+  }
+
+  @JsonProperty(index = 30)
+  public int getCurrentVersion() {
+    return dn.getCurrentVersion();
+  }
+
+  @JsonProperty(index = 35)
+  public HddsProtos.NodeOperationalState getOpState() {
+    return opState;
+  }
+
+  @JsonProperty(index = 40)
+  public HddsProtos.NodeOperationalState getPersistedOpState() {
+    return dn.getPersistedOpState();
+  }
+
+  @JsonProperty(index = 45)
+  public long getPersistedOpStateExpiryEpochSec() {
+    return dn.getPersistedOpStateExpiryEpochSec();
+  }
+
+  @JsonProperty(index = 50)
+  public HddsProtos.NodeState getHealthState() {
+    return healthState;
+  }
+
+  @JsonProperty(index = 55)
+  public boolean isDecommissioned() {
+    return dn.isDecommissioned();
+  }
+
+  @JsonProperty(index = 60)
+  public boolean isMaintenance() {
+    return dn.isMaintenance();
+  }
+
+  @JsonProperty(index = 65)
+  public int getLevel() {
+    return dn.getLevel();
+  }
+
+  @JsonProperty(index = 70)
+  public int getCost() {
+    return dn.getCost();
+  }
+
+  @JsonProperty(index = 75)
+  public int getNumOfLeaves() {
+    return dn.getNumOfLeaves();
+  }
+
+  @JsonProperty(index = 80)
+  public String getNetworkFullPath() {
+    return dn.getNetworkFullPath();
+  }
+
+  @JsonProperty(index = 85)
+  public String getNetworkLocation() {
+    return dn.getNetworkLocation();
+  }
+
+  @JsonProperty(index = 90)
+  public String getNetworkName() {
+    return dn.getNetworkName();
+  }
+
+  @JsonProperty(index = 95)
+  public Long getUsed() {
+    return used;
+  }
+
+  @JsonProperty(index = 100)
+  public Long getCapacity() {
+    return capacity;
+  }
+
+  @JsonProperty(index = 105)
+  public Double getPercentUsed() {
+    return percentUsed;
+  }
+
+  @JsonIgnore
+  public DatanodeDetails getDatanodeDetails() {
+    return dn;
+  }
+}

hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java

@@ -17,10 +17,10 @@
 
 package org.apache.hadoop.hdds.scm.cli.datanode;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
 import com.google.common.base.Strings;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 import java.util.UUID;
@@ -88,26 +88,23 @@ public void execute(ScmClient scmClient) throws IOException {
     pipelines = scmClient.listPipelines();
     if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getNodeId())) {
       HddsProtos.Node node = scmClient.queryNode(UUID.fromString(exclusiveNodeOptions.getNodeId()));
-      DatanodeWithAttributes dwa = new DatanodeWithAttributes(DatanodeDetails
-          .getFromProtoBuf(node.getNodeID()),
-          node.getNodeOperationalStates(0),
-          node.getNodeStates(0));
-
+      BasicDatanodeInfo singleNodeInfo = new BasicDatanodeInfo(DatanodeDetails.getFromProtoBuf(node.getNodeID()),
+          node.getNodeOperationalStates(0), node.getNodeStates(0));
       if (json) {
-        List<DatanodeWithAttributes> singleList = Collections.singletonList(dwa);
-        System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(singleList));
+        List<BasicDatanodeInfo> dtoList = Collections.singletonList(singleNodeInfo);
+        System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList));
       } else {
-        printDatanodeInfo(dwa);
+        printDatanodeInfo(singleNodeInfo);
       }
       return;
     }
-    Stream<DatanodeWithAttributes> allNodes = getAllNodes(scmClient).stream();
+    Stream<BasicDatanodeInfo> allNodes = getAllNodes(scmClient).stream();
     if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getIp())) {
-      allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress()
+      allNodes = allNodes.filter(p -> p.getIpAddress()
           .compareToIgnoreCase(exclusiveNodeOptions.getIp()) == 0);
     }
     if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getHostname())) {
-      allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName()
+      allNodes = allNodes.filter(p -> p.getHostName()
           .compareToIgnoreCase(exclusiveNodeOptions.getHostname()) == 0);
     }
     if (!Strings.isNullOrEmpty(nodeOperationalState)) {
@@ -124,42 +121,49 @@ public void execute(ScmClient scmClient) throws IOException {
     }
 
     if (json) {
-      List<DatanodeWithAttributes> datanodeList = allNodes.collect(
-              Collectors.toList());
-      System.out.println(
-              JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
+      List<BasicDatanodeInfo> datanodeList = allNodes.collect(Collectors.toList());
+      System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
     } else {
       allNodes.forEach(this::printDatanodeInfo);
     }
   }
 
-  private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
+  private List<BasicDatanodeInfo> getAllNodes(ScmClient scmClient)
       throws IOException {
 
     // If sorting is requested
     if (exclusiveNodeOptions != null && (exclusiveNodeOptions.mostUsed || exclusiveNodeOptions.leastUsed)) {
       boolean sortByMostUsed = exclusiveNodeOptions.mostUsed;
-      List<HddsProtos.DatanodeUsageInfoProto> usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, 
-          Integer.MAX_VALUE);
+      List<HddsProtos.DatanodeUsageInfoProto> usageInfos;
+      try {
+        usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, Integer.MAX_VALUE);
+      } catch (Exception e) {
+        System.err.println("Failed to get datanode usage info: " + e.getMessage());
+        return Collections.emptyList();
+      }
 
       return usageInfos.stream()
           .map(p -> {
-            String uuidStr = p.getNode().getUuid();
-            UUID parsedUuid = UUID.fromString(uuidStr);
-
             try {
+              String uuidStr = p.getNode().getUuid();
+              UUID parsedUuid = UUID.fromString(uuidStr);
               HddsProtos.Node node = scmClient.queryNode(parsedUuid);
               long capacity = p.getCapacity();
               long used = capacity - p.getRemaining();
               double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0;
-              return new DatanodeWithAttributes(
+              return new BasicDatanodeInfo(
                   DatanodeDetails.getFromProtoBuf(node.getNodeID()),
                   node.getNodeOperationalStates(0),
                   node.getNodeStates(0),
                   used,
                   capacity,
                   percentUsed);
-            } catch (IOException e) {
+            } catch (Exception e) {
+              String reason = "Could not process info for an unknown datanode";
+              if (p != null && p.getNode() != null && !Strings.isNullOrEmpty(p.getNode().getUuid())) {
+                reason = "Could not process node info for " + p.getNode().getUuid();
+              }
+              System.err.printf("Error: %s: %s%n", reason, e.getMessage());
               return null;
             }
           })
@@ -170,17 +174,16 @@ private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
     List<HddsProtos.Node> nodes = scmClient.queryNode(null,
         null, HddsProtos.QueryScope.CLUSTER, "");
 
-    return nodes.stream()
-        .map(p -> new DatanodeWithAttributes(
+    return nodes.stream().map(p -> new BasicDatanodeInfo(
             DatanodeDetails.getFromProtoBuf(p.getNodeID()),
             p.getNodeOperationalStates(0), p.getNodeStates(0)))
-        .sorted((o1, o2) -> o1.healthState.compareTo(o2.healthState))
+        .sorted(Comparator.comparing(BasicDatanodeInfo::getHealthState))
         .collect(Collectors.toList());
   }
 
-  private void printDatanodeInfo(DatanodeWithAttributes dna) {
+  private void printDatanodeInfo(BasicDatanodeInfo dn) {
     StringBuilder pipelineListInfo = new StringBuilder();
-    DatanodeDetails datanode = dna.getDatanodeDetails();
+    DatanodeDetails datanode = dn.getDatanodeDetails();
     int relatedPipelineNum = 0;
     if (!pipelines.isEmpty()) {
       List<Pipeline> relatedPipelines = pipelines.stream().filter(
@@ -206,72 +209,14 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) {
         " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
         + "/" + datanode.getHostName() + "/" + relatedPipelineNum +
         " pipelines)");
-    System.out.println("Operational State: " + dna.getOpState());
-    System.out.println("Health State: " + dna.getHealthState());
+    System.out.println("Operational State: " + dn.getOpState());
+    System.out.println("Health State: " + dn.getHealthState());
     System.out.println("Related pipelines:\n" + pipelineListInfo);
 
-    if (dna.getUsed() != null && dna.getCapacity() != null && dna.getUsed() >= 0 && dna.getCapacity() > 0) {
-      System.out.println("Capacity: " + dna.getCapacity());
-      System.out.println("Used: " + dna.getUsed());
-      System.out.printf("Percentage Used : %.2f%%%n%n", dna.getPercentUsed());
-    }
-  }
-
-  private static class DatanodeWithAttributes {
-    private DatanodeDetails datanodeDetails;
-    private HddsProtos.NodeOperationalState operationalState;
-    private HddsProtos.NodeState healthState;
-    @JsonInclude(JsonInclude.Include.NON_NULL)
-    private Long used = null;
-    @JsonInclude(JsonInclude.Include.NON_NULL)
-    private Long capacity = null;
-    @JsonInclude(JsonInclude.Include.NON_NULL)
-    private Double percentUsed = null;
-
-    DatanodeWithAttributes(DatanodeDetails dn,
-        HddsProtos.NodeOperationalState opState,
-        HddsProtos.NodeState healthState) {
-      this.datanodeDetails = dn;
-      this.operationalState = opState;
-      this.healthState = healthState;
-    }
-
-    DatanodeWithAttributes(DatanodeDetails dn,
-                           HddsProtos.NodeOperationalState opState,
-                           HddsProtos.NodeState healthState,
-                           long used,
-                           long capacity,
-                           double percentUsed) {
-      this.datanodeDetails = dn;
-      this.operationalState = opState;
-      this.healthState = healthState;
-      this.used = used;
-      this.capacity = capacity;
-      this.percentUsed = percentUsed;
-    }
-
-    public DatanodeDetails getDatanodeDetails() {
-      return datanodeDetails;
-    }
-
-    public HddsProtos.NodeOperationalState getOpState() {
-      return operationalState;
-    }
-
-    public HddsProtos.NodeState getHealthState() {
-      return healthState;
-    }
-
-    public Long getUsed() {
-      return used;
-    }
-
-    public Long getCapacity() {
-      return capacity;
-    }
-
-    public Double getPercentUsed() {
-      return percentUsed;
+    if (dn.getUsed() != null && dn.getCapacity() != null && dn.getUsed() >= 0 && dn.getCapacity() > 0) {
+      System.out.println("Capacity: " + dn.getCapacity());
+      System.out.println("Used: " + dn.getUsed());
+      System.out.printf("Percentage Used : %.2f%%%n%n", dn.getPercentUsed());
     }
   }
 }

hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java

@@ -207,9 +207,8 @@ public void testDataNodeByUuidOutput()
     assertEquals(1, root.size(), "Expected 1 node in JSON output");
 
     JsonNode node = root.get(0);
-    assertTrue(node.has("datanodeDetails"), "Missing datanodeDetails");
     String opState = node.get("opState").asText();
-    String uuid = node.get("datanodeDetails").get("uuid").asText();
+    String uuid = node.get("id").asText();
 
     assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got: " + opState);
     assertEquals(nodes.get(0).getNodeID().getUuid(), uuid,
@@ -365,8 +364,6 @@ private List<HddsProtos.Node> getNodeDetails() {
       dnd.setIpAddress("1.2.3." + i + 1);
       dnd.setNetworkLocation("/default");
       dnd.setNetworkName("host" + i);
-      dnd.addPorts(HddsProtos.Port.newBuilder()
-          .setName("ratis").setValue(5678).build());
       dnd.setUuid(UUID.randomUUID().toString());
 
       HddsProtos.Node.Builder builder  = HddsProtos.Node.newBuilder();