From 5a8e7be13bf8710b370dd92e07c0dfcb0ad82b0f Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 29 May 2025 18:39:16 +0530 Subject: [PATCH 01/13] HDDS-13101. Remove duplicate information in datanode list output --- .../scm/cli/datanode/ListInfoSubcommand.java | 100 ++++++++++++++++-- .../cli/datanode/TestListInfoSubcommand.java | 5 +- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 06bdaf718346..33370706ac53 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -17,6 +17,8 @@ package org.apache.hadoop.hdds.scm.cli.datanode; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.google.common.base.Strings; import java.io.IOException; import java.util.Collections; @@ -107,11 +109,11 @@ public void execute(ScmClient scmClient) throws IOException { .compareToIgnoreCase(hostname) == 0); } if (!Strings.isNullOrEmpty(nodeOperationalState)) { - allNodes = allNodes.filter(p -> p.getOpState().toString() + allNodes = allNodes.filter(p -> p.getOpState() .compareToIgnoreCase(nodeOperationalState) == 0); } if (!Strings.isNullOrEmpty(nodeState)) { - allNodes = allNodes.filter(p -> p.getHealthState().toString() + allNodes = allNodes.filter(p -> p.getHealthState() .compareToIgnoreCase(nodeState) == 0); } @@ -171,7 +173,19 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) { System.out.println("Related pipelines:\n" + pipelineListInfo); } + @JsonPropertyOrder({ + "id", + "hostname", + "ipAddress", + "healthState", + "opState", + "persistedOpStateExpiryEpochSec", + "decommissioned", + "maintenance", + "topology" + }) private static class DatanodeWithAttributes { + @JsonIgnore private DatanodeDetails datanodeDetails; private HddsProtos.NodeOperationalState operationalState; private HddsProtos.NodeState healthState; @@ -188,12 +202,86 @@ public DatanodeDetails getDatanodeDetails() { return datanodeDetails; } - public HddsProtos.NodeOperationalState getOpState() { - return operationalState; + public String getOpState() { + return operationalState.name(); } + + public String getHealthState() { + return healthState.name(); + } + + public String getId() { + return datanodeDetails.getUuid().toString(); + } + + public String getHostname() { + return datanodeDetails.getHostName(); + } + + public String getIpAddress() { + return datanodeDetails.getIpAddress(); + } + + public long getPersistedOpStateExpiryEpochSec() { + try { + return datanodeDetails.getPersistedOpStateExpiryEpochSec(); + } catch (Exception e) { + return 0L; + } + } + + public boolean isDecommissioned() { + return datanodeDetails.isDecommissioned(); + } + + public boolean isMaintenance() { + return datanodeDetails.isMaintenance(); + } + + public Topology getTopology() { + return new Topology(datanodeDetails); + } + + private static class Topology { + private final String networkLocation; + private final String networkName; + private final String networkFullPath; + private final int numOfLeaves; + private final int level; + private final int cost; - public HddsProtos.NodeState getHealthState() { - return healthState; + Topology(DatanodeDetails dn) { + this.networkLocation = dn.getNetworkLocation(); + this.networkName = dn.getNetworkName(); + this.networkFullPath = dn.getNetworkFullPath(); + this.numOfLeaves = dn.getNumOfLeaves(); + this.level = dn.getLevel(); + this.cost = dn.getCost(); + } + + public String getNetworkLocation() { + return networkLocation; + } + + public String getNetworkName() { + return networkName; + } + + public String getNetworkFullPath() { + return networkFullPath; + } + + public int getNumOfLeaves() { + return numOfLeaves; + } + + public int getLevel() { + return level; + } + + public int getCost() { + return cost; + } } } } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java index 38763ff3ae98..611c0cc80e59 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -201,9 +201,8 @@ public void testDataNodeByUuidOutput() assertEquals(1, root.size(), "Expected 1 node in JSON output"); JsonNode node = root.get(0); - assertTrue(node.has("datanodeDetails"), "Missing datanodeDetails"); String opState = node.get("opState").asText(); - String uuid = node.get("datanodeDetails").get("uuid").asText(); + String uuid = node.get("id").asText(); assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got: " + opState); assertEquals(nodes.get(0).getNodeID().getUuid(), uuid, @@ -220,8 +219,6 @@ private List getNodeDetails() { dnd.setIpAddress("1.2.3." + i + 1); dnd.setNetworkLocation("/default"); dnd.setNetworkName("host" + i); - dnd.addPorts(HddsProtos.Port.newBuilder() - .setName("ratis").setValue(5678).build()); dnd.setUuid(UUID.randomUUID().toString()); HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); From fa2d29d3abfd11d7719e63965d2de250f4ab5ec0 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 29 May 2025 20:34:05 +0530 Subject: [PATCH 02/13] Updated robot test to support new json output format --- hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot | 2 +- .../dist/src/main/smoketest/balancer/testBalancer.robot | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot index 50837672d924..9a1367808c61 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot @@ -99,7 +99,7 @@ Incomplete command List datanodes as JSON ${output} = Execute ozone admin datanode list --json | jq -r '.' - Should contain ${output} datanodeDetails + Should contain ${output} id Should contain ${output} healthState Should contain ${output} opState diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index b1801591365f..531c49ba3a6f 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -149,7 +149,7 @@ All container is closed Get Datanode Ozone Used Bytes Info [arguments] ${uuid} - ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].datanodeDetails.uuid'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add + ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].id) : .[0].ozoneUsed}'; done | jq -s add ${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value' [return] ${result} From 047942796dabfdfd15b873f256ee70cd35863bba Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 29 May 2025 21:51:48 +0530 Subject: [PATCH 03/13] fixed robot test --- .../dist/src/main/smoketest/balancer/testBalancer.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 531c49ba3a6f..1a36cb017d61 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -149,7 +149,7 @@ All container is closed Get Datanode Ozone Used Bytes Info [arguments] ${uuid} - ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].id) : .[0].ozoneUsed}'; done | jq -s add + ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add ${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value' [return] ${result} From b7dda067008c1719cca30683430c4187967862bf Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 4 Jun 2025 12:52:20 +0530 Subject: [PATCH 04/13] Using a centralized DatanodeDetails serializer --- .../scm/cli/datanode/DatanodeInfoJson.java | 133 ++++++++++++++++++ .../scm/cli/datanode/ListInfoSubcommand.java | 116 ++------------- 2 files changed, 147 insertions(+), 102 deletions(-) create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java new file mode 100644 index 000000000000..9a4e52824f2b --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli.datanode; + +import java.util.List; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; + +/** + * Represents filtered Datanode information for json use. + */ + +public class DatanodeInfoJson { + private final String id; + private final String hostName; + private final String ipAddress; + private final List ports; + private final long setupTime; + private final int currentVersion; + private final String opState; + private final long persistedOpStateExpiryEpochSec; + private final String healthState; + private final boolean decommissioned; + private final boolean maintenance; + private final int level; + private final int cost; + private final int numOfLeaves; + private final String networkFullPath; + private final String networkLocation; + private final String networkName; + + DatanodeInfoJson(ListInfoSubcommand.DatanodeWithAttributes dna) { + this.id = dna.getDatanodeDetails().getUuid().toString(); + this.ports = dna.getDatanodeDetails().getPorts(); + this.opState = String.valueOf(dna.getOpState()); + this.healthState = String.valueOf(dna.getHealthState()); + this.hostName = dna.getDatanodeDetails().getHostName(); + this.ipAddress = dna.getDatanodeDetails().getIpAddress(); + this.persistedOpStateExpiryEpochSec = dna.getDatanodeDetails().getPersistedOpStateExpiryEpochSec(); + this.decommissioned = dna.getDatanodeDetails().isDecommissioned(); + this.maintenance = dna.getDatanodeDetails().isMaintenance(); + this.level = dna.getDatanodeDetails().getLevel(); + this.cost = dna.getDatanodeDetails().getCost(); + this.numOfLeaves = dna.getDatanodeDetails().getNumOfLeaves(); + this.networkFullPath = dna.getDatanodeDetails().getNetworkFullPath(); + this.networkLocation = dna.getDatanodeDetails().getNetworkLocation(); + this.networkName = dna.getDatanodeDetails().getNetworkName(); + this.setupTime = dna.getDatanodeDetails().getSetupTime(); + this.currentVersion = dna.getDatanodeDetails().getCurrentVersion(); + } + + public String getId() { + return id; + } + + public List getPorts() { + return ports; + } + + public String getOpState() { + return opState; + } + + public String getHealthState() { + return healthState; + } + + public String getHostName() { + return hostName; + } + + public String getIpAddress() { + return ipAddress; + } + + public long getPersistedOpStateExpiryEpochSec() { + return persistedOpStateExpiryEpochSec; + } + + public boolean isDecommissioned() { + return decommissioned; + } + + public boolean isMaintenance() { + return maintenance; + } + + public int getLevel() { + return level; + } + + public int getCost() { + return cost; + } + + public int getNumOfLeaves() { + return numOfLeaves; + } + + public String getNetworkFullPath() { + return networkFullPath; + } + + public String getNetworkLocation() { + return networkLocation; + } + + public String getNetworkName() { + return networkName; + } + + public long getSetupTime() { + return setupTime; + } + + public int getCurrentVersion() { + return currentVersion; + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 33370706ac53..3a5946bce366 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -17,11 +17,8 @@ package org.apache.hadoop.hdds.scm.cli.datanode; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.google.common.base.Strings; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -92,8 +89,8 @@ public void execute(ScmClient scmClient) throws IOException { node.getNodeStates(0)); if (json) { - List singleList = Collections.singletonList(dwa); - System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(singleList)); + DatanodeInfoJson dto = new DatanodeInfoJson(dwa); + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dto)); } else { printDatanodeInfo(dwa); } @@ -109,19 +106,20 @@ public void execute(ScmClient scmClient) throws IOException { .compareToIgnoreCase(hostname) == 0); } if (!Strings.isNullOrEmpty(nodeOperationalState)) { - allNodes = allNodes.filter(p -> p.getOpState() + allNodes = allNodes.filter(p -> p.getOpState().toString() .compareToIgnoreCase(nodeOperationalState) == 0); } if (!Strings.isNullOrEmpty(nodeState)) { - allNodes = allNodes.filter(p -> p.getHealthState() + allNodes = allNodes.filter(p -> p.getHealthState().toString() .compareToIgnoreCase(nodeState) == 0); } if (json) { - List datanodeList = allNodes.collect( - Collectors.toList()); - System.out.println( - JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); + List datanodeList = allNodes + .map(DatanodeInfoJson::new) + .collect(Collectors.toList()); + + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); } else { allNodes.forEach(this::printDatanodeInfo); } @@ -173,19 +171,7 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) { System.out.println("Related pipelines:\n" + pipelineListInfo); } - @JsonPropertyOrder({ - "id", - "hostname", - "ipAddress", - "healthState", - "opState", - "persistedOpStateExpiryEpochSec", - "decommissioned", - "maintenance", - "topology" - }) - private static class DatanodeWithAttributes { - @JsonIgnore + static class DatanodeWithAttributes { private DatanodeDetails datanodeDetails; private HddsProtos.NodeOperationalState operationalState; private HddsProtos.NodeState healthState; @@ -202,86 +188,12 @@ public DatanodeDetails getDatanodeDetails() { return datanodeDetails; } - public String getOpState() { - return operationalState.name(); - } - - public String getHealthState() { - return healthState.name(); - } - - public String getId() { - return datanodeDetails.getUuid().toString(); - } - - public String getHostname() { - return datanodeDetails.getHostName(); - } - - public String getIpAddress() { - return datanodeDetails.getIpAddress(); + public HddsProtos.NodeOperationalState getOpState() { + return operationalState; } - public long getPersistedOpStateExpiryEpochSec() { - try { - return datanodeDetails.getPersistedOpStateExpiryEpochSec(); - } catch (Exception e) { - return 0L; - } - } - - public boolean isDecommissioned() { - return datanodeDetails.isDecommissioned(); - } - - public boolean isMaintenance() { - return datanodeDetails.isMaintenance(); - } - - public Topology getTopology() { - return new Topology(datanodeDetails); - } - - private static class Topology { - private final String networkLocation; - private final String networkName; - private final String networkFullPath; - private final int numOfLeaves; - private final int level; - private final int cost; - - Topology(DatanodeDetails dn) { - this.networkLocation = dn.getNetworkLocation(); - this.networkName = dn.getNetworkName(); - this.networkFullPath = dn.getNetworkFullPath(); - this.numOfLeaves = dn.getNumOfLeaves(); - this.level = dn.getLevel(); - this.cost = dn.getCost(); - } - - public String getNetworkLocation() { - return networkLocation; - } - - public String getNetworkName() { - return networkName; - } - - public String getNetworkFullPath() { - return networkFullPath; - } - - public int getNumOfLeaves() { - return numOfLeaves; - } - - public int getLevel() { - return level; - } - - public int getCost() { - return cost; - } + public HddsProtos.NodeState getHealthState() { + return healthState; } } } From 2d42ffbcb50f1f0977889011e01138129548d389 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 4 Jun 2025 13:02:24 +0530 Subject: [PATCH 05/13] Updated json output for single node --- .../hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 3a5946bce366..6e6cabe36e90 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -19,6 +19,7 @@ import com.google.common.base.Strings; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -89,8 +90,8 @@ public void execute(ScmClient scmClient) throws IOException { node.getNodeStates(0)); if (json) { - DatanodeInfoJson dto = new DatanodeInfoJson(dwa); - System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dto)); + List dtoList = Collections.singletonList(new DatanodeInfoJson(dwa)); + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList)); } else { printDatanodeInfo(dwa); } @@ -118,7 +119,6 @@ public void execute(ScmClient scmClient) throws IOException { List datanodeList = allNodes .map(DatanodeInfoJson::new) .collect(Collectors.toList()); - System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); } else { allNodes.forEach(this::printDatanodeInfo); From 8f4ae43bb3f1e33abd2526eafb140b87db678d58 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Mon, 21 Jul 2025 14:34:48 +0530 Subject: [PATCH 06/13] Fixed TestListInfoSubcommand failure and reduced redundant datanodeDetails calls --- .../scm/cli/datanode/DatanodeInfoJson.java | 53 +++++++++++++------ .../scm/cli/datanode/ListInfoSubcommand.java | 4 -- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java index 9a4e52824f2b..70fc5d0ff0b0 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.cli.datanode; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.List; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -42,25 +43,35 @@ public class DatanodeInfoJson { private final String networkFullPath; private final String networkLocation; private final String networkName; + @JsonInclude(JsonInclude.Include.NON_NULL) + private Long used = null; + @JsonInclude(JsonInclude.Include.NON_NULL) + private Long capacity = null; + @JsonInclude(JsonInclude.Include.NON_NULL) + private Double percentUsed = null; DatanodeInfoJson(ListInfoSubcommand.DatanodeWithAttributes dna) { - this.id = dna.getDatanodeDetails().getUuid().toString(); - this.ports = dna.getDatanodeDetails().getPorts(); + DatanodeDetails dn = dna.getDatanodeDetails(); + this.id = dn.getUuid().toString(); + this.ports = dn.getPorts(); this.opState = String.valueOf(dna.getOpState()); this.healthState = String.valueOf(dna.getHealthState()); - this.hostName = dna.getDatanodeDetails().getHostName(); - this.ipAddress = dna.getDatanodeDetails().getIpAddress(); - this.persistedOpStateExpiryEpochSec = dna.getDatanodeDetails().getPersistedOpStateExpiryEpochSec(); - this.decommissioned = dna.getDatanodeDetails().isDecommissioned(); - this.maintenance = dna.getDatanodeDetails().isMaintenance(); - this.level = dna.getDatanodeDetails().getLevel(); - this.cost = dna.getDatanodeDetails().getCost(); - this.numOfLeaves = dna.getDatanodeDetails().getNumOfLeaves(); - this.networkFullPath = dna.getDatanodeDetails().getNetworkFullPath(); - this.networkLocation = dna.getDatanodeDetails().getNetworkLocation(); - this.networkName = dna.getDatanodeDetails().getNetworkName(); - this.setupTime = dna.getDatanodeDetails().getSetupTime(); - this.currentVersion = dna.getDatanodeDetails().getCurrentVersion(); + this.hostName = dn.getHostName(); + this.ipAddress = dn.getIpAddress(); + this.persistedOpStateExpiryEpochSec = dn.getPersistedOpStateExpiryEpochSec(); + this.decommissioned = dn.isDecommissioned(); + this.maintenance = dn.isMaintenance(); + this.level = dn.getLevel(); + this.cost = dn.getCost(); + this.numOfLeaves = dn.getNumOfLeaves(); + this.networkFullPath = dn.getNetworkFullPath(); + this.networkLocation = dn.getNetworkLocation(); + this.networkName = dn.getNetworkName(); + this.setupTime = dn.getSetupTime(); + this.currentVersion = dn.getCurrentVersion(); + this.used = dna.getUsed(); + this.capacity = dna.getCapacity(); + this.percentUsed = dna.getPercentUsed(); } public String getId() { @@ -130,4 +141,16 @@ public long getSetupTime() { public int getCurrentVersion() { return currentVersion; } + + public Long getUsed() { + return used; + } + + public Long getCapacity() { + return capacity; + } + + public Double getPercentUsed() { + return percentUsed; + } } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index f02f8729e572..1a8893b29a65 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.cli.datanode; -import com.fasterxml.jackson.annotation.JsonInclude; import com.google.common.base.Strings; import java.io.IOException; import java.util.Collections; @@ -224,11 +223,8 @@ static class DatanodeWithAttributes { private DatanodeDetails datanodeDetails; private HddsProtos.NodeOperationalState operationalState; private HddsProtos.NodeState healthState; - @JsonInclude(JsonInclude.Include.NON_NULL) private Long used = null; - @JsonInclude(JsonInclude.Include.NON_NULL) private Long capacity = null; - @JsonInclude(JsonInclude.Include.NON_NULL) private Double percentUsed = null; DatanodeWithAttributes(DatanodeDetails dn, From 249ecb4b83fa87eb984e89b611b566fff4af725e Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 24 Jul 2025 17:30:25 +0530 Subject: [PATCH 07/13] Replace opState with persistedOpState and bypass DatanodeWithAttributes in DatanodeInfoJson --- ...foJson.java => BasicDatanodeInfoJson.java} | 72 +++++++----- .../scm/cli/datanode/ListInfoSubcommand.java | 106 ++++-------------- .../cli/datanode/TestListInfoSubcommand.java | 8 +- 3 files changed, 71 insertions(+), 115 deletions(-) rename hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/{DatanodeInfoJson.java => BasicDatanodeInfoJson.java} (63%) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java similarity index 63% rename from hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java rename to hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java index 70fc5d0ff0b0..cfaf8f9bccf2 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeInfoJson.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java @@ -17,24 +17,26 @@ package org.apache.hadoop.hdds.scm.cli.datanode; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import java.util.List; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; /** * Represents filtered Datanode information for json use. */ -public class DatanodeInfoJson { +public class BasicDatanodeInfoJson { private final String id; private final String hostName; private final String ipAddress; private final List ports; private final long setupTime; private final int currentVersion; - private final String opState; + private final String persistedOpState; private final long persistedOpStateExpiryEpochSec; - private final String healthState; + private final HddsProtos.NodeState healthState; private final boolean decommissioned; private final boolean maintenance; private final int level; @@ -49,29 +51,36 @@ public class DatanodeInfoJson { private Long capacity = null; @JsonInclude(JsonInclude.Include.NON_NULL) private Double percentUsed = null; - - DatanodeInfoJson(ListInfoSubcommand.DatanodeWithAttributes dna) { - DatanodeDetails dn = dna.getDatanodeDetails(); - this.id = dn.getUuid().toString(); - this.ports = dn.getPorts(); - this.opState = String.valueOf(dna.getOpState()); - this.healthState = String.valueOf(dna.getHealthState()); - this.hostName = dn.getHostName(); - this.ipAddress = dn.getIpAddress(); - this.persistedOpStateExpiryEpochSec = dn.getPersistedOpStateExpiryEpochSec(); - this.decommissioned = dn.isDecommissioned(); - this.maintenance = dn.isMaintenance(); - this.level = dn.getLevel(); - this.cost = dn.getCost(); - this.numOfLeaves = dn.getNumOfLeaves(); - this.networkFullPath = dn.getNetworkFullPath(); - this.networkLocation = dn.getNetworkLocation(); - this.networkName = dn.getNetworkName(); - this.setupTime = dn.getSetupTime(); - this.currentVersion = dn.getCurrentVersion(); - this.used = dna.getUsed(); - this.capacity = dna.getCapacity(); - this.percentUsed = dna.getPercentUsed(); + @JsonIgnore + private DatanodeDetails dn; + + public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeState healthState) { + this.dn = dnDetails; + this.id = dnDetails.getUuid().toString(); + this.ports = dnDetails.getPorts(); + this.persistedOpState = String.valueOf(dnDetails.getPersistedOpState()); + this.healthState = healthState; + this.hostName = dnDetails.getHostName(); + this.ipAddress = dnDetails.getIpAddress(); + this.persistedOpStateExpiryEpochSec = dnDetails.getPersistedOpStateExpiryEpochSec(); + this.decommissioned = dnDetails.isDecommissioned(); + this.maintenance = dnDetails.isMaintenance(); + this.level = dnDetails.getLevel(); + this.cost = dnDetails.getCost(); + this.numOfLeaves = dnDetails.getNumOfLeaves(); + this.networkFullPath = dnDetails.getNetworkFullPath(); + this.networkLocation = dnDetails.getNetworkLocation(); + this.networkName = dnDetails.getNetworkName(); + this.setupTime = dnDetails.getSetupTime(); + this.currentVersion = dnDetails.getCurrentVersion(); + } + + public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeState healthState, + long used, long capacity, double percentUsed) { + this(dnDetails, healthState); + this.used = used; + this.capacity = capacity; + this.percentUsed = percentUsed; } public String getId() { @@ -82,11 +91,11 @@ public List getPorts() { return ports; } - public String getOpState() { - return opState; + public String getPersistedOpState() { + return persistedOpState; } - public String getHealthState() { + public HddsProtos.NodeState getHealthState() { return healthState; } @@ -153,4 +162,9 @@ public Long getCapacity() { public Double getPercentUsed() { return percentUsed; } + + @JsonIgnore + public DatanodeDetails getDatanodeDetails() { + return dn; + } } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 1a8893b29a65..e1f67b85ead0 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -90,30 +90,27 @@ public void execute(ScmClient scmClient) throws IOException { pipelines = scmClient.listPipelines(); if (!Strings.isNullOrEmpty(nodeSelectionMixin.getNodeId())) { HddsProtos.Node node = scmClient.queryNode(UUID.fromString(nodeSelectionMixin.getNodeId())); - DatanodeWithAttributes dwa = new DatanodeWithAttributes(DatanodeDetails - .getFromProtoBuf(node.getNodeID()), - node.getNodeOperationalStates(0), - node.getNodeStates(0)); - + BasicDatanodeInfoJson singleNodeInfo = new BasicDatanodeInfoJson( + DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeStates(0)); if (json) { - List dtoList = Collections.singletonList(new DatanodeInfoJson(dwa)); + List dtoList = Collections.singletonList(singleNodeInfo); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList)); } else { - printDatanodeInfo(dwa); + printDatanodeInfo(singleNodeInfo); } return; } - Stream allNodes = getAllNodes(scmClient).stream(); + Stream allNodes = getAllNodes(scmClient).stream(); if (!Strings.isNullOrEmpty(nodeSelectionMixin.getIp())) { - allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress() + allNodes = allNodes.filter(p -> p.getIpAddress() .compareToIgnoreCase(nodeSelectionMixin.getIp()) == 0); } if (!Strings.isNullOrEmpty(nodeSelectionMixin.getHostname())) { - allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName() + allNodes = allNodes.filter(p -> p.getHostName() .compareToIgnoreCase(nodeSelectionMixin.getHostname()) == 0); } if (!Strings.isNullOrEmpty(nodeOperationalState)) { - allNodes = allNodes.filter(p -> p.getOpState().toString() + allNodes = allNodes.filter(p -> p.getPersistedOpState() .compareToIgnoreCase(nodeOperationalState) == 0); } if (!Strings.isNullOrEmpty(nodeState)) { @@ -126,16 +123,14 @@ public void execute(ScmClient scmClient) throws IOException { } if (json) { - List datanodeList = allNodes - .map(DatanodeInfoJson::new) - .collect(Collectors.toList()); + List datanodeList = allNodes.collect(Collectors.toList()); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); } else { allNodes.forEach(this::printDatanodeInfo); } } - private List getAllNodes(ScmClient scmClient) + private List getAllNodes(ScmClient scmClient) throws IOException { // If sorting is requested @@ -154,9 +149,8 @@ private List getAllNodes(ScmClient scmClient) long capacity = p.getCapacity(); long used = capacity - p.getRemaining(); double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0; - return new DatanodeWithAttributes( + return new BasicDatanodeInfoJson( DatanodeDetails.getFromProtoBuf(node.getNodeID()), - node.getNodeOperationalStates(0), node.getNodeStates(0), used, capacity, @@ -173,16 +167,15 @@ private List getAllNodes(ScmClient scmClient) null, HddsProtos.QueryScope.CLUSTER, ""); return nodes.stream() - .map(p -> new DatanodeWithAttributes( - DatanodeDetails.getFromProtoBuf(p.getNodeID()), - p.getNodeOperationalStates(0), p.getNodeStates(0))) - .sorted((o1, o2) -> o1.healthState.compareTo(o2.healthState)) + .map(p -> new BasicDatanodeInfoJson( + DatanodeDetails.getFromProtoBuf(p.getNodeID()), p.getNodeStates(0))) + .sorted((o1, o2) -> o1.getHealthState().compareTo(o2.getHealthState())) .collect(Collectors.toList()); } - private void printDatanodeInfo(DatanodeWithAttributes dna) { + private void printDatanodeInfo(BasicDatanodeInfoJson dn) { StringBuilder pipelineListInfo = new StringBuilder(); - DatanodeDetails datanode = dna.getDatanodeDetails(); + DatanodeDetails datanode = dn.getDatanodeDetails(); int relatedPipelineNum = 0; if (!pipelines.isEmpty()) { List relatedPipelines = pipelines.stream().filter( @@ -208,69 +201,14 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) { " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress() + "/" + datanode.getHostName() + "/" + relatedPipelineNum + " pipelines)"); - System.out.println("Operational State: " + dna.getOpState()); - System.out.println("Health State: " + dna.getHealthState()); + System.out.println("Operational State: " + dn.getPersistedOpState()); + System.out.println("Health State: " + dn.getHealthState()); System.out.println("Related pipelines:\n" + pipelineListInfo); - if (dna.getUsed() != null && dna.getCapacity() != null && dna.getUsed() >= 0 && dna.getCapacity() > 0) { - System.out.println("Capacity: " + dna.getCapacity()); - System.out.println("Used: " + dna.getUsed()); - System.out.printf("Percentage Used : %.2f%%%n%n", dna.getPercentUsed()); - } - } - - static class DatanodeWithAttributes { - private DatanodeDetails datanodeDetails; - private HddsProtos.NodeOperationalState operationalState; - private HddsProtos.NodeState healthState; - private Long used = null; - private Long capacity = null; - private Double percentUsed = null; - - DatanodeWithAttributes(DatanodeDetails dn, - HddsProtos.NodeOperationalState opState, - HddsProtos.NodeState healthState) { - this.datanodeDetails = dn; - this.operationalState = opState; - this.healthState = healthState; - } - - DatanodeWithAttributes(DatanodeDetails dn, - HddsProtos.NodeOperationalState opState, - HddsProtos.NodeState healthState, - long used, - long capacity, - double percentUsed) { - this.datanodeDetails = dn; - this.operationalState = opState; - this.healthState = healthState; - this.used = used; - this.capacity = capacity; - this.percentUsed = percentUsed; - } - - public DatanodeDetails getDatanodeDetails() { - return datanodeDetails; - } - - public HddsProtos.NodeOperationalState getOpState() { - return operationalState; - } - - public HddsProtos.NodeState getHealthState() { - return healthState; - } - - public Long getUsed() { - return used; - } - - public Long getCapacity() { - return capacity; - } - - public Double getPercentUsed() { - return percentUsed; + if (dn.getUsed() != null && dn.getCapacity() != null && dn.getUsed() >= 0 && dn.getCapacity() > 0) { + System.out.println("Capacity: " + dn.getCapacity()); + System.out.println("Used: " + dn.getUsed()); + System.out.printf("Percentage Used : %.2f%%%n%n", dn.getPercentUsed()); } } } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java index a6ad7c7bb5ec..00819e0d7581 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -135,7 +135,7 @@ public void testDataNodeOperationalStateAndHealthIncludedInOutput() List operationalStates = new ArrayList<>(); for (JsonNode node : root) { healthStates.add(node.get("healthState").asText()); - operationalStates.add(node.get("opState").asText()); + operationalStates.add(node.get("persistedOpState").asText()); } // Check expected operational states are present @@ -206,7 +206,7 @@ public void testDataNodeByUuidOutput() assertEquals(1, root.size(), "Expected 1 node in JSON output"); JsonNode node = root.get(0); - String opState = node.get("opState").asText(); + String opState = node.get("persistedOpState").asText(); String uuid = node.get("id").asText(); assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got: " + opState); @@ -335,18 +335,22 @@ private List getNodeDetails() { if (i == 0) { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.IN_SERVICE); + dnd.setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE); builder.addNodeStates(HddsProtos.NodeState.STALE); } else if (i == 1) { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.DECOMMISSIONING); + dnd.setPersistedOpState(HddsProtos.NodeOperationalState.DECOMMISSIONING); builder.addNodeStates(HddsProtos.NodeState.DEAD); } else if (i == 2) { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.IN_SERVICE); + dnd.setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE); builder.addNodeStates(HddsProtos.NodeState.HEALTHY_READONLY); } else { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.IN_SERVICE); + dnd.setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE); builder.addNodeStates(HddsProtos.NodeState.HEALTHY); } builder.setNodeID(dnd.build()); From 040c91d29c9dc74996638c69b3f1df86123a8670 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 24 Jul 2025 21:29:09 +0530 Subject: [PATCH 08/13] Fixed datanode.robot test --- hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot index 56b5309237ac..f08d710f3870 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot @@ -101,7 +101,7 @@ List datanodes as JSON ${output} = Execute ozone admin datanode list --json | jq -r '.' Should contain ${output} id Should contain ${output} healthState - Should contain ${output} opState + Should contain ${output} persistedOpState Get usage info as JSON ${output} = Execute ozone admin datanode usageinfo -m --json | jq -r '.' From 9c5c5faf6df8aab3aa62736e9a6366b797413425 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Fri, 25 Jul 2025 20:43:39 +0530 Subject: [PATCH 09/13] Fixed testBalancer.robot test --- .../hdds/scm/cli/datanode/BasicDatanodeInfoJson.java | 6 +++--- .../hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java | 2 +- .../dist/src/main/smoketest/balancer/testBalancer.robot | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java index cfaf8f9bccf2..7a5cca64e6b1 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java @@ -34,7 +34,7 @@ public class BasicDatanodeInfoJson { private final List ports; private final long setupTime; private final int currentVersion; - private final String persistedOpState; + private final HddsProtos.NodeOperationalState persistedOpState; private final long persistedOpStateExpiryEpochSec; private final HddsProtos.NodeState healthState; private final boolean decommissioned; @@ -58,7 +58,7 @@ public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeState hea this.dn = dnDetails; this.id = dnDetails.getUuid().toString(); this.ports = dnDetails.getPorts(); - this.persistedOpState = String.valueOf(dnDetails.getPersistedOpState()); + this.persistedOpState = dnDetails.getPersistedOpState(); this.healthState = healthState; this.hostName = dnDetails.getHostName(); this.ipAddress = dnDetails.getIpAddress(); @@ -91,7 +91,7 @@ public List getPorts() { return ports; } - public String getPersistedOpState() { + public HddsProtos.NodeOperationalState getPersistedOpState() { return persistedOpState; } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index e1f67b85ead0..35c9ac6c5298 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -110,7 +110,7 @@ public void execute(ScmClient scmClient) throws IOException { .compareToIgnoreCase(nodeSelectionMixin.getHostname()) == 0); } if (!Strings.isNullOrEmpty(nodeOperationalState)) { - allNodes = allNodes.filter(p -> p.getPersistedOpState() + allNodes = allNodes.filter(p -> p.getPersistedOpState().toString() .compareToIgnoreCase(nodeOperationalState) == 0); } if (!Strings.isNullOrEmpty(nodeState)) { diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 0436348239be..078c3f5eddc4 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -43,11 +43,15 @@ Prepare For Tests Datanode In Maintenance Mode ${result} = Execute ozone admin datanode maintenance ${HOST} Should Contain ${result} Entering maintenance mode on datanode - ${result} = Execute ozone admin datanode list | grep "Operational State:*" - Wait Until Keyword Succeeds 30sec 5sec Should contain ${result} ENTERING_MAINTENANCE + Wait Until Keyword Succeeds 30sec 5sec Datanode Operational State Should Be ENTERING_MAINTENANCE Wait Until Keyword Succeeds 3min 10sec Related pipelines are closed Sleep 60000ms +Datanode Operational State Should Be + [Arguments] ${expected_state} + ${result} = Execute ozone admin datanode list | grep "Operational State:*" + Should Contain ${result} ${expected_state} + Related pipelines are closed ${result} = Execute ozone admin datanode list | awk -v RS= '{$1=$1}1'|grep MAINT | sed -e 's/^.*pipelines: \\(.*\\)$/\\1/' Should Contain Any ${result} CLOSED No related pipelines or the node is not in Healthy state. From a209c463e6cc4ad56ea1644d6fb7514b98ca92dd Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Fri, 25 Jul 2025 23:56:40 +0530 Subject: [PATCH 10/13] Included both opState and persistedOpState in json and updated tests --- .../scm/cli/datanode/BasicDatanodeInfoJson.java | 14 +++++++++++--- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 10 ++++++---- .../scm/cli/datanode/TestListInfoSubcommand.java | 8 ++------ .../src/main/smoketest/admincli/datanode.robot | 1 + .../src/main/smoketest/balancer/testBalancer.robot | 8 ++------ 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java index 7a5cca64e6b1..b4038944c9e7 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java @@ -35,6 +35,7 @@ public class BasicDatanodeInfoJson { private final long setupTime; private final int currentVersion; private final HddsProtos.NodeOperationalState persistedOpState; + private final HddsProtos.NodeOperationalState opState; private final long persistedOpStateExpiryEpochSec; private final HddsProtos.NodeState healthState; private final boolean decommissioned; @@ -54,11 +55,13 @@ public class BasicDatanodeInfoJson { @JsonIgnore private DatanodeDetails dn; - public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeState healthState) { + public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState healthState) { this.dn = dnDetails; this.id = dnDetails.getUuid().toString(); this.ports = dnDetails.getPorts(); this.persistedOpState = dnDetails.getPersistedOpState(); + this.opState = opState; this.healthState = healthState; this.hostName = dnDetails.getHostName(); this.ipAddress = dnDetails.getIpAddress(); @@ -75,9 +78,10 @@ public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeState hea this.currentVersion = dnDetails.getCurrentVersion(); } - public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeState healthState, + public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState healthState, long used, long capacity, double percentUsed) { - this(dnDetails, healthState); + this(dnDetails, opState, healthState); this.used = used; this.capacity = capacity; this.percentUsed = percentUsed; @@ -94,6 +98,10 @@ public List getPorts() { public HddsProtos.NodeOperationalState getPersistedOpState() { return persistedOpState; } + + public HddsProtos.NodeOperationalState getOpState() { + return opState; + } public HddsProtos.NodeState getHealthState() { return healthState; diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 35c9ac6c5298..d59f4f92f145 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -91,7 +91,7 @@ public void execute(ScmClient scmClient) throws IOException { if (!Strings.isNullOrEmpty(nodeSelectionMixin.getNodeId())) { HddsProtos.Node node = scmClient.queryNode(UUID.fromString(nodeSelectionMixin.getNodeId())); BasicDatanodeInfoJson singleNodeInfo = new BasicDatanodeInfoJson( - DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeStates(0)); + DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeOperationalStates(0), node.getNodeStates(0)); if (json) { List dtoList = Collections.singletonList(singleNodeInfo); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList)); @@ -110,7 +110,7 @@ public void execute(ScmClient scmClient) throws IOException { .compareToIgnoreCase(nodeSelectionMixin.getHostname()) == 0); } if (!Strings.isNullOrEmpty(nodeOperationalState)) { - allNodes = allNodes.filter(p -> p.getPersistedOpState().toString() + allNodes = allNodes.filter(p -> p.getOpState().toString() .compareToIgnoreCase(nodeOperationalState) == 0); } if (!Strings.isNullOrEmpty(nodeState)) { @@ -151,6 +151,7 @@ private List getAllNodes(ScmClient scmClient) double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0; return new BasicDatanodeInfoJson( DatanodeDetails.getFromProtoBuf(node.getNodeID()), + node.getNodeOperationalStates(0), node.getNodeStates(0), used, capacity, @@ -168,7 +169,8 @@ private List getAllNodes(ScmClient scmClient) return nodes.stream() .map(p -> new BasicDatanodeInfoJson( - DatanodeDetails.getFromProtoBuf(p.getNodeID()), p.getNodeStates(0))) + DatanodeDetails.getFromProtoBuf(p.getNodeID()), + p.getNodeOperationalStates(0), p.getNodeStates(0))) .sorted((o1, o2) -> o1.getHealthState().compareTo(o2.getHealthState())) .collect(Collectors.toList()); } @@ -201,7 +203,7 @@ private void printDatanodeInfo(BasicDatanodeInfoJson dn) { " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress() + "/" + datanode.getHostName() + "/" + relatedPipelineNum + " pipelines)"); - System.out.println("Operational State: " + dn.getPersistedOpState()); + System.out.println("Operational State: " + dn.getOpState()); System.out.println("Health State: " + dn.getHealthState()); System.out.println("Related pipelines:\n" + pipelineListInfo); diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java index 00819e0d7581..a6ad7c7bb5ec 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -135,7 +135,7 @@ public void testDataNodeOperationalStateAndHealthIncludedInOutput() List operationalStates = new ArrayList<>(); for (JsonNode node : root) { healthStates.add(node.get("healthState").asText()); - operationalStates.add(node.get("persistedOpState").asText()); + operationalStates.add(node.get("opState").asText()); } // Check expected operational states are present @@ -206,7 +206,7 @@ public void testDataNodeByUuidOutput() assertEquals(1, root.size(), "Expected 1 node in JSON output"); JsonNode node = root.get(0); - String opState = node.get("persistedOpState").asText(); + String opState = node.get("opState").asText(); String uuid = node.get("id").asText(); assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got: " + opState); @@ -335,22 +335,18 @@ private List getNodeDetails() { if (i == 0) { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.IN_SERVICE); - dnd.setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE); builder.addNodeStates(HddsProtos.NodeState.STALE); } else if (i == 1) { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.DECOMMISSIONING); - dnd.setPersistedOpState(HddsProtos.NodeOperationalState.DECOMMISSIONING); builder.addNodeStates(HddsProtos.NodeState.DEAD); } else if (i == 2) { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.IN_SERVICE); - dnd.setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE); builder.addNodeStates(HddsProtos.NodeState.HEALTHY_READONLY); } else { builder.addNodeOperationalStates( HddsProtos.NodeOperationalState.IN_SERVICE); - dnd.setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE); builder.addNodeStates(HddsProtos.NodeState.HEALTHY); } builder.setNodeID(dnd.build()); diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot index f08d710f3870..2795ae5ac4a6 100644 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot @@ -101,6 +101,7 @@ List datanodes as JSON ${output} = Execute ozone admin datanode list --json | jq -r '.' Should contain ${output} id Should contain ${output} healthState + Should contain ${output} opState Should contain ${output} persistedOpState Get usage info as JSON diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 078c3f5eddc4..0436348239be 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -43,15 +43,11 @@ Prepare For Tests Datanode In Maintenance Mode ${result} = Execute ozone admin datanode maintenance ${HOST} Should Contain ${result} Entering maintenance mode on datanode - Wait Until Keyword Succeeds 30sec 5sec Datanode Operational State Should Be ENTERING_MAINTENANCE + ${result} = Execute ozone admin datanode list | grep "Operational State:*" + Wait Until Keyword Succeeds 30sec 5sec Should contain ${result} ENTERING_MAINTENANCE Wait Until Keyword Succeeds 3min 10sec Related pipelines are closed Sleep 60000ms -Datanode Operational State Should Be - [Arguments] ${expected_state} - ${result} = Execute ozone admin datanode list | grep "Operational State:*" - Should Contain ${result} ${expected_state} - Related pipelines are closed ${result} = Execute ozone admin datanode list | awk -v RS= '{$1=$1}1'|grep MAINT | sed -e 's/^.*pipelines: \\(.*\\)$/\\1/' Should Contain Any ${result} CLOSED No related pipelines or the node is not in Healthy state. From 8bb258490c4268bb5e2f3ad1275c7dbe6c67e5a8 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 31 Jul 2025 09:20:20 +0530 Subject: [PATCH 11/13] Used DatanodeDetails directly in BasicDatanodeInfo to avoid field copies --- ...deInfoJson.java => BasicDatanodeInfo.java} | 102 ++++++++---------- .../scm/cli/datanode/ListInfoSubcommand.java | 18 ++-- 2 files changed, 53 insertions(+), 67 deletions(-) rename hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/{BasicDatanodeInfoJson.java => BasicDatanodeInfo.java} (57%) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java similarity index 57% rename from hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java rename to hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java index b4038944c9e7..92992d7f6eb0 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfoJson.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.List; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -26,61 +27,46 @@ /** * Represents filtered Datanode information for json use. */ - -public class BasicDatanodeInfoJson { - private final String id; - private final String hostName; - private final String ipAddress; - private final List ports; - private final long setupTime; - private final int currentVersion; - private final HddsProtos.NodeOperationalState persistedOpState; - private final HddsProtos.NodeOperationalState opState; - private final long persistedOpStateExpiryEpochSec; - private final HddsProtos.NodeState healthState; - private final boolean decommissioned; - private final boolean maintenance; - private final int level; - private final int cost; - private final int numOfLeaves; - private final String networkFullPath; - private final String networkLocation; - private final String networkName; +@JsonPropertyOrder({ + "id", + "hostName", + "ipAddress", + "ports", + "setupTime", + "currentVersion", + "persistedOpState", + "opState", + "persistedOpStateExpiryEpochSec", + "healthState", + "decommissioned", + "maintenance", + "level", + "cost", + "numOfLeaves", + "networkFullPath", + "networkLocation", + "networkName" +}) +public class BasicDatanodeInfo { @JsonInclude(JsonInclude.Include.NON_NULL) private Long used = null; @JsonInclude(JsonInclude.Include.NON_NULL) private Long capacity = null; @JsonInclude(JsonInclude.Include.NON_NULL) private Double percentUsed = null; - @JsonIgnore - private DatanodeDetails dn; + private final DatanodeDetails dn; + private final HddsProtos.NodeOperationalState opState; + private final HddsProtos.NodeState healthState; - public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, + public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, HddsProtos.NodeState healthState) { this.dn = dnDetails; - this.id = dnDetails.getUuid().toString(); - this.ports = dnDetails.getPorts(); - this.persistedOpState = dnDetails.getPersistedOpState(); this.opState = opState; this.healthState = healthState; - this.hostName = dnDetails.getHostName(); - this.ipAddress = dnDetails.getIpAddress(); - this.persistedOpStateExpiryEpochSec = dnDetails.getPersistedOpStateExpiryEpochSec(); - this.decommissioned = dnDetails.isDecommissioned(); - this.maintenance = dnDetails.isMaintenance(); - this.level = dnDetails.getLevel(); - this.cost = dnDetails.getCost(); - this.numOfLeaves = dnDetails.getNumOfLeaves(); - this.networkFullPath = dnDetails.getNetworkFullPath(); - this.networkLocation = dnDetails.getNetworkLocation(); - this.networkName = dnDetails.getNetworkName(); - this.setupTime = dnDetails.getSetupTime(); - this.currentVersion = dnDetails.getCurrentVersion(); } - public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, - HddsProtos.NodeState healthState, - long used, long capacity, double percentUsed) { + public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState healthState, long used, long capacity, double percentUsed) { this(dnDetails, opState, healthState); this.used = used; this.capacity = capacity; @@ -88,15 +74,15 @@ public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperation } public String getId() { - return id; + return dn.getUuidString(); } public List getPorts() { - return ports; + return dn.getPorts(); } public HddsProtos.NodeOperationalState getPersistedOpState() { - return persistedOpState; + return dn.getPersistedOpState(); } public HddsProtos.NodeOperationalState getOpState() { @@ -108,55 +94,55 @@ public HddsProtos.NodeState getHealthState() { } public String getHostName() { - return hostName; + return dn.getHostName(); } public String getIpAddress() { - return ipAddress; + return dn.getIpAddress(); } public long getPersistedOpStateExpiryEpochSec() { - return persistedOpStateExpiryEpochSec; + return dn.getPersistedOpStateExpiryEpochSec(); } public boolean isDecommissioned() { - return decommissioned; + return dn.isDecommissioned(); } public boolean isMaintenance() { - return maintenance; + return dn.isMaintenance(); } public int getLevel() { - return level; + return dn.getLevel(); } public int getCost() { - return cost; + return dn.getCost(); } public int getNumOfLeaves() { - return numOfLeaves; + return dn.getNumOfLeaves(); } public String getNetworkFullPath() { - return networkFullPath; + return dn.getNetworkFullPath(); } public String getNetworkLocation() { - return networkLocation; + return dn.getNetworkLocation(); } public String getNetworkName() { - return networkName; + return dn.getNetworkName(); } public long getSetupTime() { - return setupTime; + return dn.getSetupTime(); } public int getCurrentVersion() { - return currentVersion; + return dn.getCurrentVersion(); } public Long getUsed() { diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index c76c5f0feb37..9271fcca0dcc 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -87,17 +87,17 @@ public void execute(ScmClient scmClient) throws IOException { pipelines = scmClient.listPipelines(); if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getNodeId())) { HddsProtos.Node node = scmClient.queryNode(UUID.fromString(exclusiveNodeOptions.getNodeId())); - BasicDatanodeInfoJson singleNodeInfo = new BasicDatanodeInfoJson( - DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeOperationalStates(0), node.getNodeStates(0)); + BasicDatanodeInfo singleNodeInfo = new BasicDatanodeInfo(DatanodeDetails.getFromProtoBuf(node.getNodeID()), + node.getNodeOperationalStates(0), node.getNodeStates(0)); if (json) { - List dtoList = Collections.singletonList(singleNodeInfo); + List dtoList = Collections.singletonList(singleNodeInfo); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList)); } else { printDatanodeInfo(singleNodeInfo); } return; } - Stream allNodes = getAllNodes(scmClient).stream(); + Stream allNodes = getAllNodes(scmClient).stream(); if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getIp())) { allNodes = allNodes.filter(p -> p.getIpAddress() .compareToIgnoreCase(exclusiveNodeOptions.getIp()) == 0); @@ -120,14 +120,14 @@ public void execute(ScmClient scmClient) throws IOException { } if (json) { - List datanodeList = allNodes.collect(Collectors.toList()); + List datanodeList = allNodes.collect(Collectors.toList()); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList)); } else { allNodes.forEach(this::printDatanodeInfo); } } - private List getAllNodes(ScmClient scmClient) + private List getAllNodes(ScmClient scmClient) throws IOException { // If sorting is requested @@ -146,7 +146,7 @@ private List getAllNodes(ScmClient scmClient) long capacity = p.getCapacity(); long used = capacity - p.getRemaining(); double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0; - return new BasicDatanodeInfoJson( + return new BasicDatanodeInfo( DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeOperationalStates(0), node.getNodeStates(0), @@ -165,14 +165,14 @@ private List getAllNodes(ScmClient scmClient) null, HddsProtos.QueryScope.CLUSTER, ""); return nodes.stream() - .map(p -> new BasicDatanodeInfoJson( + .map(p -> new BasicDatanodeInfo( DatanodeDetails.getFromProtoBuf(p.getNodeID()), p.getNodeOperationalStates(0), p.getNodeStates(0))) .sorted((o1, o2) -> o1.getHealthState().compareTo(o2.getHealthState())) .collect(Collectors.toList()); } - private void printDatanodeInfo(BasicDatanodeInfoJson dn) { + private void printDatanodeInfo(BasicDatanodeInfo dn) { StringBuilder pipelineListInfo = new StringBuilder(); DatanodeDetails datanode = dn.getDatanodeDetails(); int relatedPipelineNum = 0; From fa73f054c8fda748fd6687820b952063266dfaa7 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Fri, 1 Aug 2025 13:17:42 +0530 Subject: [PATCH 12/13] Used JsonProperty index, passed limit to server side when sorting, updated Exception handling --- .../scm/cli/datanode/BasicDatanodeInfo.java | 89 ++++++++++--------- .../scm/cli/datanode/ListInfoSubcommand.java | 30 ++++--- .../cli/datanode/TestListInfoSubcommand.java | 2 +- 3 files changed, 65 insertions(+), 56 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java index 92992d7f6eb0..4a925ffad5ff 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -27,26 +27,6 @@ /** * Represents filtered Datanode information for json use. */ -@JsonPropertyOrder({ - "id", - "hostName", - "ipAddress", - "ports", - "setupTime", - "currentVersion", - "persistedOpState", - "opState", - "persistedOpStateExpiryEpochSec", - "healthState", - "decommissioned", - "maintenance", - "level", - "cost", - "numOfLeaves", - "networkFullPath", - "networkLocation", - "networkName" -}) public class BasicDatanodeInfo { @JsonInclude(JsonInclude.Include.NON_NULL) private Long used = null; @@ -73,86 +53,107 @@ public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalSt this.percentUsed = percentUsed; } + @JsonProperty(index = 5) public String getId() { return dn.getUuidString(); } + @JsonProperty(index = 10) + public String getHostName() { + return dn.getHostName(); + } + + @JsonProperty(index = 15) + public String getIpAddress() { + return dn.getIpAddress(); + } + + @JsonProperty(index = 20) public List getPorts() { return dn.getPorts(); } - - public HddsProtos.NodeOperationalState getPersistedOpState() { - return dn.getPersistedOpState(); + + @JsonProperty(index = 25) + public long getSetupTime() { + return dn.getSetupTime(); + } + + @JsonProperty(index = 30) + public int getCurrentVersion() { + return dn.getCurrentVersion(); } + @JsonProperty(index = 35) public HddsProtos.NodeOperationalState getOpState() { return opState; } - - public HddsProtos.NodeState getHealthState() { - return healthState; - } - - public String getHostName() { - return dn.getHostName(); - } - - public String getIpAddress() { - return dn.getIpAddress(); + + @JsonProperty(index = 40) + public HddsProtos.NodeOperationalState getPersistedOpState() { + return dn.getPersistedOpState(); } - + + @JsonProperty(index = 45) public long getPersistedOpStateExpiryEpochSec() { return dn.getPersistedOpStateExpiryEpochSec(); } + @JsonProperty(index = 50) + public HddsProtos.NodeState getHealthState() { + return healthState; + } + + @JsonProperty(index = 55) public boolean isDecommissioned() { return dn.isDecommissioned(); } + @JsonProperty(index = 60) public boolean isMaintenance() { return dn.isMaintenance(); } + @JsonProperty(index = 65) public int getLevel() { return dn.getLevel(); } + @JsonProperty(index = 70) public int getCost() { return dn.getCost(); } + @JsonProperty(index = 75) public int getNumOfLeaves() { return dn.getNumOfLeaves(); } + @JsonProperty(index = 80) public String getNetworkFullPath() { return dn.getNetworkFullPath(); } + @JsonProperty(index = 85) public String getNetworkLocation() { return dn.getNetworkLocation(); } + @JsonProperty(index = 90) public String getNetworkName() { return dn.getNetworkName(); } - public long getSetupTime() { - return dn.getSetupTime(); - } - - public int getCurrentVersion() { - return dn.getCurrentVersion(); - } - + @JsonProperty(index = 95) public Long getUsed() { return used; } + @JsonProperty(index = 100) public Long getCapacity() { return capacity; } + @JsonProperty(index = 105) public Double getPercentUsed() { return percentUsed; } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 9271fcca0dcc..3c16260fdddf 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -20,6 +20,7 @@ import com.google.common.base.Strings; import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.UUID; @@ -114,10 +115,6 @@ public void execute(ScmClient scmClient) throws IOException { allNodes = allNodes.filter(p -> p.getHealthState().toString() .compareToIgnoreCase(nodeState) == 0); } - - if (!listLimitOptions.isAll()) { - allNodes = allNodes.limit(listLimitOptions.getLimit()); - } if (json) { List datanodeList = allNodes.collect(Collectors.toList()); @@ -129,19 +126,24 @@ public void execute(ScmClient scmClient) throws IOException { private List getAllNodes(ScmClient scmClient) throws IOException { + int limit = listLimitOptions.getLimit(); // If sorting is requested if (exclusiveNodeOptions != null && (exclusiveNodeOptions.mostUsed || exclusiveNodeOptions.leastUsed)) { boolean sortByMostUsed = exclusiveNodeOptions.mostUsed; - List usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, - Integer.MAX_VALUE); + List usageInfos; + try { + usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, limit); + } catch (Exception e) { + System.err.println("Failed to get datanode usage info: " + e.getMessage()); + return Collections.emptyList(); + } return usageInfos.stream() .map(p -> { - String uuidStr = p.getNode().getUuid(); - UUID parsedUuid = UUID.fromString(uuidStr); - try { + String uuidStr = p.getNode().getUuid(); + UUID parsedUuid = UUID.fromString(uuidStr); HddsProtos.Node node = scmClient.queryNode(parsedUuid); long capacity = p.getCapacity(); long used = capacity - p.getRemaining(); @@ -153,7 +155,12 @@ private List getAllNodes(ScmClient scmClient) used, capacity, percentUsed); - } catch (IOException e) { + } catch (Exception e) { + String reason = "Could not process info for an unknown datanode"; + if (p != null && p.getNode() != null && !Strings.isNullOrEmpty(p.getNode().getUuid())) { + reason = "Could not process node info for " + p.getNode().getUuid(); + } + System.err.printf("Error: %s: %s%n", reason, e.getMessage()); return null; } }) @@ -165,10 +172,11 @@ private List getAllNodes(ScmClient scmClient) null, HddsProtos.QueryScope.CLUSTER, ""); return nodes.stream() + .limit(limit) .map(p -> new BasicDatanodeInfo( DatanodeDetails.getFromProtoBuf(p.getNodeID()), p.getNodeOperationalStates(0), p.getNodeStates(0))) - .sorted((o1, o2) -> o1.getHealthState().compareTo(o2.getHealthState())) + .sorted(Comparator.comparing(BasicDatanodeInfo::getHealthState)) .collect(Collectors.toList()); } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java index fd6834068eef..f815d9330a71 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -245,7 +245,7 @@ public void testUsedOrderingAndOutput( Collections.reverse(usageList); // For most-used only } - when(scmClient.getDatanodeUsageInfo(mostUsed, Integer.MAX_VALUE)).thenReturn(usageList); + when(scmClient.getDatanodeUsageInfo(mostUsed, 100)).thenReturn(usageList); when(scmClient.listPipelines()).thenReturn(new ArrayList<>()); // ----- JSON output test ----- From aa3889b71187b16a0cf79a3f4889e77899e70a9b Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Fri, 1 Aug 2025 23:15:56 +0530 Subject: [PATCH 13/13] Reverted to original limit flow --- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 11 ++++++----- .../hdds/scm/cli/datanode/TestListInfoSubcommand.java | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 3c16260fdddf..49fb032d6ce1 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -115,6 +115,10 @@ public void execute(ScmClient scmClient) throws IOException { allNodes = allNodes.filter(p -> p.getHealthState().toString() .compareToIgnoreCase(nodeState) == 0); } + + if (!listLimitOptions.isAll()) { + allNodes = allNodes.limit(listLimitOptions.getLimit()); + } if (json) { List datanodeList = allNodes.collect(Collectors.toList()); @@ -126,14 +130,13 @@ public void execute(ScmClient scmClient) throws IOException { private List getAllNodes(ScmClient scmClient) throws IOException { - int limit = listLimitOptions.getLimit(); // If sorting is requested if (exclusiveNodeOptions != null && (exclusiveNodeOptions.mostUsed || exclusiveNodeOptions.leastUsed)) { boolean sortByMostUsed = exclusiveNodeOptions.mostUsed; List usageInfos; try { - usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, limit); + usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed, Integer.MAX_VALUE); } catch (Exception e) { System.err.println("Failed to get datanode usage info: " + e.getMessage()); return Collections.emptyList(); @@ -171,9 +174,7 @@ private List getAllNodes(ScmClient scmClient) List nodes = scmClient.queryNode(null, null, HddsProtos.QueryScope.CLUSTER, ""); - return nodes.stream() - .limit(limit) - .map(p -> new BasicDatanodeInfo( + return nodes.stream().map(p -> new BasicDatanodeInfo( DatanodeDetails.getFromProtoBuf(p.getNodeID()), p.getNodeOperationalStates(0), p.getNodeStates(0))) .sorted(Comparator.comparing(BasicDatanodeInfo::getHealthState)) diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java index f815d9330a71..fd6834068eef 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -245,7 +245,7 @@ public void testUsedOrderingAndOutput( Collections.reverse(usageList); // For most-used only } - when(scmClient.getDatanodeUsageInfo(mostUsed, 100)).thenReturn(usageList); + when(scmClient.getDatanodeUsageInfo(mostUsed, Integer.MAX_VALUE)).thenReturn(usageList); when(scmClient.listPipelines()).thenReturn(new ArrayList<>()); // ----- JSON output test -----