Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.hdds.scm.cli.datanode;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;

/**
* Represents filtered Datanode information for json use.
*/

public class BasicDatanodeInfoJson {
Comment thread
errose28 marked this conversation as resolved.
Outdated
private final String id;
private final String hostName;
private final String ipAddress;
private final List<DatanodeDetails.Port> ports;
private final long setupTime;
private final int currentVersion;
private final HddsProtos.NodeOperationalState persistedOpState;
private final HddsProtos.NodeOperationalState opState;
private final long persistedOpStateExpiryEpochSec;
private final HddsProtos.NodeState healthState;
private final boolean decommissioned;
private final boolean maintenance;
private final int level;
private final int cost;
private final int numOfLeaves;
private final String networkFullPath;
private final String networkLocation;
private final String networkName;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long used = null;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long capacity = null;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Double percentUsed = null;
@JsonIgnore
private DatanodeDetails dn;

public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState) {
this.dn = dnDetails;
this.id = dnDetails.getUuid().toString();
Comment thread
errose28 marked this conversation as resolved.
Outdated
this.ports = dnDetails.getPorts();
this.persistedOpState = dnDetails.getPersistedOpState();
this.opState = opState;
this.healthState = healthState;
this.hostName = dnDetails.getHostName();
this.ipAddress = dnDetails.getIpAddress();
this.persistedOpStateExpiryEpochSec = dnDetails.getPersistedOpStateExpiryEpochSec();
this.decommissioned = dnDetails.isDecommissioned();
this.maintenance = dnDetails.isMaintenance();
this.level = dnDetails.getLevel();
this.cost = dnDetails.getCost();
this.numOfLeaves = dnDetails.getNumOfLeaves();
this.networkFullPath = dnDetails.getNetworkFullPath();
this.networkLocation = dnDetails.getNetworkLocation();
this.networkName = dnDetails.getNetworkName();
this.setupTime = dnDetails.getSetupTime();
this.currentVersion = dnDetails.getCurrentVersion();
}

public BasicDatanodeInfoJson(DatanodeDetails dnDetails, HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState,
long used, long capacity, double percentUsed) {
this(dnDetails, opState, healthState);
this.used = used;
this.capacity = capacity;
this.percentUsed = percentUsed;
}

public String getId() {
return id;
}

public List<DatanodeDetails.Port> getPorts() {
return ports;
}

public HddsProtos.NodeOperationalState getPersistedOpState() {
return persistedOpState;
}

public HddsProtos.NodeOperationalState getOpState() {
return opState;
}

public HddsProtos.NodeState getHealthState() {
return healthState;
}

public String getHostName() {
return hostName;
}

public String getIpAddress() {
return ipAddress;
}

public long getPersistedOpStateExpiryEpochSec() {
return persistedOpStateExpiryEpochSec;
}

public boolean isDecommissioned() {
return decommissioned;
}

public boolean isMaintenance() {
return maintenance;
}

public int getLevel() {
return level;
}

public int getCost() {
return cost;
}

public int getNumOfLeaves() {
return numOfLeaves;
}

public String getNetworkFullPath() {
return networkFullPath;
}

public String getNetworkLocation() {
return networkLocation;
}

public String getNetworkName() {
return networkName;
}

public long getSetupTime() {
return setupTime;
}

public int getCurrentVersion() {
return currentVersion;
}

public Long getUsed() {
return used;
}

public Long getCapacity() {
return capacity;
}

public Double getPercentUsed() {
return percentUsed;
}

@JsonIgnore
public DatanodeDetails getDatanodeDetails() {
return dn;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.hadoop.hdds.scm.cli.datanode;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.google.common.base.Strings;
import java.io.IOException;
import java.util.Collections;
Expand Down Expand Up @@ -91,26 +90,23 @@ public void execute(ScmClient scmClient) throws IOException {
pipelines = scmClient.listPipelines();
if (!Strings.isNullOrEmpty(nodeSelectionMixin.getNodeId())) {
HddsProtos.Node node = scmClient.queryNode(UUID.fromString(nodeSelectionMixin.getNodeId()));
DatanodeWithAttributes dwa = new DatanodeWithAttributes(DatanodeDetails
.getFromProtoBuf(node.getNodeID()),
node.getNodeOperationalStates(0),
node.getNodeStates(0));

BasicDatanodeInfoJson singleNodeInfo = new BasicDatanodeInfoJson(
DatanodeDetails.getFromProtoBuf(node.getNodeID()), node.getNodeOperationalStates(0), node.getNodeStates(0));
if (json) {
List<DatanodeWithAttributes> singleList = Collections.singletonList(dwa);
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(singleList));
List<BasicDatanodeInfoJson> dtoList = Collections.singletonList(singleNodeInfo);
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList));
} else {
printDatanodeInfo(dwa);
printDatanodeInfo(singleNodeInfo);
}
return;
}
Stream<DatanodeWithAttributes> allNodes = getAllNodes(scmClient).stream();
Stream<BasicDatanodeInfoJson> allNodes = getAllNodes(scmClient).stream();
if (!Strings.isNullOrEmpty(nodeSelectionMixin.getIp())) {
allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress()
allNodes = allNodes.filter(p -> p.getIpAddress()
.compareToIgnoreCase(nodeSelectionMixin.getIp()) == 0);
}
if (!Strings.isNullOrEmpty(nodeSelectionMixin.getHostname())) {
allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName()
allNodes = allNodes.filter(p -> p.getHostName()
.compareToIgnoreCase(nodeSelectionMixin.getHostname()) == 0);
}
if (!Strings.isNullOrEmpty(nodeOperationalState)) {
Expand All @@ -127,16 +123,14 @@ public void execute(ScmClient scmClient) throws IOException {
}

if (json) {
List<DatanodeWithAttributes> datanodeList = allNodes.collect(
Collectors.toList());
System.out.println(
JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
List<BasicDatanodeInfoJson> datanodeList = allNodes.collect(Collectors.toList());
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
} else {
allNodes.forEach(this::printDatanodeInfo);
}
}

private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
private List<BasicDatanodeInfoJson> getAllNodes(ScmClient scmClient)
throws IOException {

// If sorting is requested
Expand All @@ -155,7 +149,7 @@ private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
long capacity = p.getCapacity();
long used = capacity - p.getRemaining();
double percentUsed = (capacity > 0) ? (used * 100.0) / capacity : 0.0;
return new DatanodeWithAttributes(
return new BasicDatanodeInfoJson(
Comment thread
errose28 marked this conversation as resolved.
Outdated
DatanodeDetails.getFromProtoBuf(node.getNodeID()),
node.getNodeOperationalStates(0),
node.getNodeStates(0),
Expand All @@ -174,16 +168,16 @@ private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
null, HddsProtos.QueryScope.CLUSTER, "");

return nodes.stream()
.map(p -> new DatanodeWithAttributes(
.map(p -> new BasicDatanodeInfoJson(
DatanodeDetails.getFromProtoBuf(p.getNodeID()),
p.getNodeOperationalStates(0), p.getNodeStates(0)))
.sorted((o1, o2) -> o1.healthState.compareTo(o2.healthState))
.sorted((o1, o2) -> o1.getHealthState().compareTo(o2.getHealthState()))
.collect(Collectors.toList());
}

private void printDatanodeInfo(DatanodeWithAttributes dna) {
private void printDatanodeInfo(BasicDatanodeInfoJson dn) {
StringBuilder pipelineListInfo = new StringBuilder();
DatanodeDetails datanode = dna.getDatanodeDetails();
DatanodeDetails datanode = dn.getDatanodeDetails();
int relatedPipelineNum = 0;
if (!pipelines.isEmpty()) {
List<Pipeline> relatedPipelines = pipelines.stream().filter(
Expand All @@ -209,72 +203,14 @@ private void printDatanodeInfo(DatanodeWithAttributes dna) {
" (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
+ "/" + datanode.getHostName() + "/" + relatedPipelineNum +
" pipelines)");
System.out.println("Operational State: " + dna.getOpState());
System.out.println("Health State: " + dna.getHealthState());
System.out.println("Operational State: " + dn.getOpState());
System.out.println("Health State: " + dn.getHealthState());
System.out.println("Related pipelines:\n" + pipelineListInfo);

if (dna.getUsed() != null && dna.getCapacity() != null && dna.getUsed() >= 0 && dna.getCapacity() > 0) {
System.out.println("Capacity: " + dna.getCapacity());
System.out.println("Used: " + dna.getUsed());
System.out.printf("Percentage Used : %.2f%%%n%n", dna.getPercentUsed());
}
}

private static class DatanodeWithAttributes {
private DatanodeDetails datanodeDetails;
private HddsProtos.NodeOperationalState operationalState;
private HddsProtos.NodeState healthState;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long used = null;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long capacity = null;
@JsonInclude(JsonInclude.Include.NON_NULL)
private Double percentUsed = null;

DatanodeWithAttributes(DatanodeDetails dn,
HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState) {
this.datanodeDetails = dn;
this.operationalState = opState;
this.healthState = healthState;
}

DatanodeWithAttributes(DatanodeDetails dn,
HddsProtos.NodeOperationalState opState,
HddsProtos.NodeState healthState,
long used,
long capacity,
double percentUsed) {
this.datanodeDetails = dn;
this.operationalState = opState;
this.healthState = healthState;
this.used = used;
this.capacity = capacity;
this.percentUsed = percentUsed;
}

public DatanodeDetails getDatanodeDetails() {
return datanodeDetails;
}

public HddsProtos.NodeOperationalState getOpState() {
return operationalState;
}

public HddsProtos.NodeState getHealthState() {
return healthState;
}

public Long getUsed() {
return used;
}

public Long getCapacity() {
return capacity;
}

public Double getPercentUsed() {
return percentUsed;
if (dn.getUsed() != null && dn.getCapacity() != null && dn.getUsed() >= 0 && dn.getCapacity() > 0) {
System.out.println("Capacity: " + dn.getCapacity());
System.out.println("Used: " + dn.getUsed());
System.out.printf("Percentage Used : %.2f%%%n%n", dn.getPercentUsed());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,8 @@ public void testDataNodeByUuidOutput()
assertEquals(1, root.size(), "Expected 1 node in JSON output");

JsonNode node = root.get(0);
assertTrue(node.has("datanodeDetails"), "Missing datanodeDetails");
String opState = node.get("opState").asText();
String uuid = node.get("datanodeDetails").get("uuid").asText();
String uuid = node.get("id").asText();

assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got: " + opState);
assertEquals(nodes.get(0).getNodeID().getUuid(), uuid,
Expand Down Expand Up @@ -330,8 +329,6 @@ private List<HddsProtos.Node> getNodeDetails() {
dnd.setIpAddress("1.2.3." + i + 1);
dnd.setNetworkLocation("/default");
dnd.setNetworkName("host" + i);
dnd.addPorts(HddsProtos.Port.newBuilder()
.setName("ratis").setValue(5678).build());
dnd.setUuid(UUID.randomUUID().toString());

HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder();
Expand Down
3 changes: 2 additions & 1 deletion hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,10 @@ Incomplete command

List datanodes as JSON
${output} = Execute ozone admin datanode list --json | jq -r '.'
Should contain ${output} datanodeDetails
Should contain ${output} id
Should contain ${output} healthState
Should contain ${output} opState
Should contain ${output} persistedOpState

Get usage info as JSON
${output} = Execute ozone admin datanode usageinfo -m --json | jq -r '.'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ All container is closed

Get Datanode Ozone Used Bytes Info
[arguments] ${uuid}
${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].datanodeDetails.uuid'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add
${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add
${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value'
[return] ${result}

Expand Down