Skip to content

HDDS-13002. Use DatanodeID in ContainerBalancerTask #8421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szetszwo merged 2 commits into from
May 11, 2025
+22 −22
Merged

HDDS-13002. Use DatanodeID in ContainerBalancerTask #8421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1ca5d76
HDDS-13002. Use DatanodeID in ContainerBalancerTask
nandakumar131 May 9, 2025
480fc95
Fixed CI failure
nandakumar131 May 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions ...cm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
Expand All @@ -49,6 +48,7 @@
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.scm.PlacementPolicyValidateProxy;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
Expand Down Expand Up @@ -329,9 +329,9 @@ private ContainerBalancerTaskIterationStatusInfo getIterationStatistic(Integer i
IterationResult currentIterationResult,
long iterationDuration) {
String currentIterationResultName = currentIterationResult == null ? null : currentIterationResult.name();
Map<UUID, Long> sizeEnteringDataToNodes =
Map<DatanodeID, Long> sizeEnteringDataToNodes =
convertToNodeIdToTrafficMap(findTargetStrategy.getSizeEnteringNodes());
Map<UUID, Long> sizeLeavingDataFromNodes =
Map<DatanodeID, Long> sizeLeavingDataFromNodes =
convertToNodeIdToTrafficMap(findSourceStrategy.getSizeLeavingNodes());
IterationInfo iterationInfo = new IterationInfo(
iterationNumber,
Expand All @@ -345,8 +345,8 @@ private ContainerBalancerTaskIterationStatusInfo getIterationStatistic(Integer i
return new ContainerBalancerTaskIterationStatusInfo(iterationInfo, containerMoveInfo, dataMoveInfo);
}

private DataMoveInfo getDataMoveInfo(String currentIterationResultName, Map<UUID, Long> sizeEnteringDataToNodes,
Map<UUID, Long> sizeLeavingDataFromNodes) {
private DataMoveInfo getDataMoveInfo(String currentIterationResultName, Map<DatanodeID, Long> sizeEnteringDataToNodes,
Map<DatanodeID, Long> sizeLeavingDataFromNodes) {
if (currentIterationResultName == null) {
// For unfinished iteration
return new DataMoveInfo(
Expand All @@ -366,15 +366,15 @@ private DataMoveInfo getDataMoveInfo(String currentIterationResultName, Map<UUID
}
}

private Map<UUID, Long> convertToNodeIdToTrafficMap(Map<DatanodeDetails, Long> nodeTrafficMap) {
private Map<DatanodeID, Long> convertToNodeIdToTrafficMap(Map<DatanodeDetails, Long> nodeTrafficMap) {
return nodeTrafficMap
.entrySet()
.stream()
.filter(Objects::nonNull)
.filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0)
.collect(
Collectors.toMap(
entry -> entry.getKey().getUuid(),
entry -> entry.getKey().getID(),
Map.Entry::getValue
)
);
Expand Down Expand Up @@ -556,13 +556,13 @@ private boolean initializeIteration() {
overUtilizedNodes.forEach(entry -> {
LOG.debug("Datanode {} {} is Over-Utilized.",
entry.getDatanodeDetails().getHostName(),
entry.getDatanodeDetails().getUuid());
entry.getDatanodeDetails().getID());
});

underUtilizedNodes.forEach(entry -> {
LOG.debug("Datanode {} {} is Under-Utilized.",
entry.getDatanodeDetails().getHostName(),
entry.getDatanodeDetails().getUuid());
entry.getDatanodeDetails().getID());
});
}

Expand Down
8 changes: 4 additions & 4 deletions ...g/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTaskIterationStatusInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;

/**
Expand Down Expand Up @@ -110,15 +110,15 @@ public long getContainerMovesTimeout() {
* Get a map of the node IDs and the corresponding data sizes moved to each node.
* @return nodeId to size entering from node map
*/
public Map<UUID, Long> getSizeEnteringNodes() {
public Map<DatanodeID, Long> getSizeEnteringNodes() {
return dataMoveInfo.getSizeEnteringNodes();
}

/**
* Get a map of the node IDs and the corresponding data sizes moved from each node.
* @return nodeId to size leaving from node map
*/
public Map<UUID, Long> getSizeLeavingNodes() {
public Map<DatanodeID, Long> getSizeLeavingNodes() {
return dataMoveInfo.getSizeLeavingNodes();
}

Expand Down Expand Up @@ -160,7 +160,7 @@ public StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStat
* @return node transfer info proto representation
*/
private List<StorageContainerLocationProtocolProtos.NodeTransferInfoProto> mapToProtoNodeTransferInfo(
Map<UUID, Long> nodes
Map<DatanodeID, Long> nodes
) {
return nodes.entrySet()
.stream()
Expand Down
14 changes: 7 additions & 7 deletions .../server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/DataMoveInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,22 @@
package org.apache.hadoop.hdds.scm.container.balancer;

import java.util.Map;
import java.util.UUID;
import org.apache.hadoop.hdds.protocol.DatanodeID;

/**
* Information about the process of moving data.
*/
public class DataMoveInfo {
private final long sizeScheduledForMove;
private final long dataSizeMoved;
private final Map<UUID, Long> sizeEnteringNodes;
private final Map<UUID, Long> sizeLeavingNodes;
private final Map<DatanodeID, Long> sizeEnteringNodes;
private final Map<DatanodeID, Long> sizeLeavingNodes;

public DataMoveInfo(
long sizeScheduledForMove,
long dataSizeMoved,
Map<UUID, Long> sizeEnteringNodes,
Map<UUID, Long> sizeLeavingNodes) {
Map<DatanodeID, Long> sizeEnteringNodes,
Map<DatanodeID, Long> sizeLeavingNodes) {
this.sizeScheduledForMove = sizeScheduledForMove;
this.dataSizeMoved = dataSizeMoved;
this.sizeEnteringNodes = sizeEnteringNodes;
Expand All @@ -48,11 +48,11 @@ public long getDataSizeMoved() {
return dataSizeMoved;
}

public Map<UUID, Long> getSizeEnteringNodes() {
public Map<DatanodeID, Long> getSizeEnteringNodes() {
return sizeEnteringNodes;
}

public Map<UUID, Long> getSizeLeavingNodes() {
public Map<DatanodeID, Long> getSizeLeavingNodes() {
return sizeLeavingNodes;
}
}
4 changes: 2 additions & 2 deletions ...t/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerStatusInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@

import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.commons.math3.util.ArithmeticUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ozone.test.LambdaTestUtils;
Expand Down Expand Up @@ -229,7 +229,7 @@ private void verifyStartedEmptyIteration(
assertTrue(iteration.getSizeLeavingNodes().isEmpty());
}

private static Long getTotalMovedData(Map<UUID, Long> iteration) {
private static Long getTotalMovedData(Map<DatanodeID, Long> iteration) {
return iteration.values().stream().reduce(0L, ArithmeticUtils::addAndCheck);
}

Expand Down