Skip to content

HDDS-13101. Remove duplicate information in datanode list output #8523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
errose28 merged 15 commits into from
Aug 7, 2025
Merged

HDDS-13101. Remove duplicate information in datanode list output #8523

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
5a8e7be
HDDS-13101. Remove duplicate information in datanode list output
May 29, 2025
fa2d29d
Updated robot test to support new json output format
May 29, 2025
0479427
fixed robot test
May 29, 2025
b7dda06
Using a centralized DatanodeDetails serializer
Jun 4, 2025
2d42ffb
Updated json output for single node
Jun 4, 2025
b435d66
Merge branch 'master' of github.com:apache/ozone into HDDS-13101
Jul 21, 2025
8f4ae43
Fixed TestListInfoSubcommand failure and reduced redundant datanodeDe…
Jul 21, 2025
249ecb4
Replace opState with persistedOpState and bypass DatanodeWithAttribut…
Jul 24, 2025
040c91d
Fixed datanode.robot test
Jul 24, 2025
9c5c5fa
Fixed testBalancer.robot test
Jul 25, 2025
a209c46
Included both opState and persistedOpState in json and updated tests
Jul 25, 2025
0d4ca66
Merge branch 'master' of github.com:apache/ozone into HDDS-13101
Jul 28, 2025
8bb2584
Used DatanodeDetails directly in BasicDatanodeInfo to avoid field copies
Jul 31, 2025
fa73f05
Used JsonProperty index, passed limit to server side when sorting, up…
Aug 1, 2025
aa3889b
Reverted to original limit flow
Aug 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 44 additions & 58 deletions ...m/cli/datanode/BasicDatanodeInfoJson.java → ...s/scm/cli/datanode/BasicDatanodeInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,84 +19,70 @@

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import java.util.List;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;

/**
* Represents filtered Datanode information for json use.
*/

public class BasicDatanodeInfoJson {
private final String id;
private final String hostName;
private final String ipAddress;
private final List<DatanodeDetails.Port> ports;
private final long setupTime;
private final int currentVersion;
private final HddsProtos.NodeOperationalState persistedOpState;
private final HddsProtos.NodeOperationalState opState;
private final long persistedOpStateExpiryEpochSec;
private final HddsProtos.NodeState healthState;
private final boolean decommissioned;
private final boolean maintenance;
private final int level;
private final int cost;
private final int numOfLeaves;
private final String networkFullPath;
private final String networkLocation;
private final String networkName;
@JsonPropertyOrder({
"id",
"hostName",
"ipAddress",
"ports",
"setupTime",
"currentVersion",
"persistedOpState",
"opState",
"persistedOpStateExpiryEpochSec",
"healthState",
"decommissioned",
"maintenance",
"level",
"cost",
"numOfLeaves",
"networkFullPath",
"networkLocation",
"networkName"
})
public class BasicDatanodeInfo {
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long used = null;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long capacity = null;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Double percentUsed = null;
@JsonIgnore
private DatanodeDetails dn;
private final DatanodeDetails dn;
private final HddsProtos.NodeOperationalState opState;
private final HddsProtos.NodeState healthState;

public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState) {
this.dn = dnDetails;
this.id = dnDetails.getUuid().toString();
this.ports = dnDetails.getPorts();
this.persistedOpState = dnDetails.getPersistedOpState();
this.opState = opState;
this.healthState = healthState;
this.hostName = dnDetails.getHostName();
this.ipAddress = dnDetails.getIpAddress();
this.persistedOpStateExpiryEpochSec = dnDetails.getPersistedOpStateExpiryEpochSec();
this.decommissioned = dnDetails.isDecommissioned();
this.maintenance = dnDetails.isMaintenance();
this.level = dnDetails.getLevel();
this.cost = dnDetails.getCost();
this.numOfLeaves = dnDetails.getNumOfLeaves();
this.networkFullPath = dnDetails.getNetworkFullPath();
this.networkLocation = dnDetails.getNetworkLocation();
this.networkName = dnDetails.getNetworkName();
this.setupTime = dnDetails.getSetupTime();
this.currentVersion = dnDetails.getCurrentVersion();
}

public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState,
long used, long capacity, double percentUsed) {
public BasicDatanodeInfo(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState, long used, long capacity, double percentUsed) {
this(dnDetails, opState, healthState);
this.used = used;
this.capacity = capacity;
this.percentUsed = percentUsed;
}

public String getId() {
return id;
return dn.getUuidString();
}

public List<DatanodeDetails.Port> getPorts() {
return ports;
return dn.getPorts();
}

public HddsProtos.NodeOperationalState getPersistedOpState() {
return persistedOpState;
return dn.getPersistedOpState();
}

public HddsProtos.NodeOperationalState getOpState() {
Expand All @@ -108,55 +94,55 @@ public HddsProtos.NodeState getHealthState() {
}

public String getHostName() {
return hostName;
return dn.getHostName();
}

public String getIpAddress() {
return ipAddress;
return dn.getIpAddress();
}

public long getPersistedOpStateExpiryEpochSec() {
return persistedOpStateExpiryEpochSec;
return dn.getPersistedOpStateExpiryEpochSec();
}

public boolean isDecommissioned() {
return decommissioned;
return dn.isDecommissioned();
}

public boolean isMaintenance() {
return maintenance;
return dn.isMaintenance();
}

public int getLevel() {
return level;
return dn.getLevel();
}

public int getCost() {
return cost;
return dn.getCost();
}

public int getNumOfLeaves() {
return numOfLeaves;
return dn.getNumOfLeaves();
}

public String getNetworkFullPath() {
return networkFullPath;
return dn.getNetworkFullPath();
}

public String getNetworkLocation() {
return networkLocation;
return dn.getNetworkLocation();
}

public String getNetworkName() {
return networkName;
return dn.getNetworkName();
}

public long getSetupTime() {
return setupTime;
return dn.getSetupTime();
}

public int getCurrentVersion() {
return currentVersion;
return dn.getCurrentVersion();
}

public Long getUsed() {
Expand Down
30 changes: 15 additions & 15 deletions ...e/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,26 +85,26 @@ static class ExclusiveNodeOptions extends NodeSelectionMixin {
@Override
public void execute(ScmClient scmClient) throws IOException {
pipelines = scmClient.listPipelines();
if (!Strings.isNullOrEmpty(nodeSelectionMixin.getNodeId())) {
HddsProtos.Node node = scmClient.queryNode(UUID.fromString(nodeSelectionMixin.getNodeId()));
BasicDatanodeInfoJson singleNodeInfo = new BasicDatanodeInfoJson(
DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeOperationalStates(0), node.getNodeStates(0));
if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getNodeId())) {
HddsProtos.Node node = scmClient.queryNode(UUID.fromString(exclusiveNodeOptions.getNodeId()));
BasicDatanodeInfo singleNodeInfo = new BasicDatanodeInfo(DatanodeDetails.getFromProtoBuf(node.getNodeID()),
node.getNodeOperationalStates(0), node.getNodeStates(0));
if (json) {
List<BasicDatanodeInfoJson> dtoList = Collections.singletonList(singleNodeInfo);
List<BasicDatanodeInfo> dtoList = Collections.singletonList(singleNodeInfo);
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList));
} else {
printDatanodeInfo(singleNodeInfo);
}
return;
}
Stream<BasicDatanodeInfoJson> allNodes = getAllNodes(scmClient).stream();
if (!Strings.isNullOrEmpty(nodeSelectionMixin.getIp())) {
Stream<BasicDatanodeInfo> allNodes = getAllNodes(scmClient).stream();
if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getIp())) {
allNodes = allNodes.filter(p -> p.getIpAddress()
.compareToIgnoreCase(nodeSelectionMixin.getIp()) == 0);
.compareToIgnoreCase(exclusiveNodeOptions.getIp()) == 0);
}
if (!Strings.isNullOrEmpty(nodeSelectionMixin.getHostname())) {
if (exclusiveNodeOptions != null && !Strings.isNullOrEmpty(exclusiveNodeOptions.getHostname())) {
allNodes = allNodes.filter(p -> p.getHostName()
.compareToIgnoreCase(nodeSelectionMixin.getHostname()) == 0);
.compareToIgnoreCase(exclusiveNodeOptions.getHostname()) == 0);
}
if (!Strings.isNullOrEmpty(nodeOperationalState)) {
allNodes = allNodes.filter(p -> p.getOpState().toString()
Expand All @@ -120,14 +120,14 @@ public void execute(ScmClient scmClient) throws IOException {
}

if (json) {
List<BasicDatanodeInfoJson> datanodeList = allNodes.collect(Collectors.toList());
List<BasicDatanodeInfo> datanodeList = allNodes.collect(Collectors.toList());
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
} else {
allNodes.forEach(this::printDatanodeInfo);
}
}

private List<BasicDatanodeInfoJson> getAllNodes(ScmClient scmClient)
private List<BasicDatanodeInfo> getAllNodes(ScmClient scmClient)
throws IOException {

// If sorting is requested
Expand All @@ -146,7 +146,7 @@ private List<BasicDatanodeInfoJson> getAllNodes(ScmClient scmClient)
long capacity = p.getCapacity();
long used = capacity - p.getRemaining();
double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0;
return new BasicDatanodeInfoJson(
return new BasicDatanodeInfo(
DatanodeDetails.getFromProtoBuf(node.getNodeID()),
node.getNodeOperationalStates(0),
node.getNodeStates(0),
Expand All @@ -165,14 +165,14 @@ private List<BasicDatanodeInfoJson> getAllNodes(ScmClient scmClient)
null, HddsProtos.QueryScope.CLUSTER, "");

return nodes.stream()
.map(p -> new BasicDatanodeInfoJson(
.map(p -> new BasicDatanodeInfo(
DatanodeDetails.getFromProtoBuf(p.getNodeID()),
p.getNodeOperationalStates(0), p.getNodeStates(0)))
.sorted((o1, o2) -> o1.getHealthState().compareTo(o2.getHealthState()))
.collect(Collectors.toList());
}

private void printDatanodeInfo(BasicDatanodeInfoJson dn) {
private void printDatanodeInfo(BasicDatanodeInfo dn) {
StringBuilder pipelineListInfo = new StringBuilder();
DatanodeDetails datanode = dn.getDatanodeDetails();
int relatedPipelineNum = 0;
Expand Down