diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java index dd2d1c578940..88657047a07a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java @@ -32,11 +32,11 @@ import org.slf4j.Logger; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** @@ -56,7 +56,7 @@ protected AbstractFindTargetGreedy( ContainerManager containerManager, PlacementPolicyValidateProxy placementPolicyValidateProxy, NodeManager nodeManager) { - sizeEnteringNode = new HashMap<>(); + sizeEnteringNode = new ConcurrentHashMap<>(); this.containerManager = containerManager; this.placementPolicyValidateProxy = placementPolicyValidateProxy; this.nodeManager = nodeManager; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java index 19a2f3c2e621..93d496519b26 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java @@ -51,10 +51,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.Queue; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -117,7 +118,7 @@ public class ContainerBalancerTask implements Runnable { private IterationResult iterationResult; private int nextIterationIndex; private boolean delayStart; - private List iterationsStatistic; + private Queue iterationsStatistic; /** * Constructs ContainerBalancerTask with the specified arguments. @@ -166,7 +167,7 @@ public ContainerBalancerTask(StorageContainerManager scm, findTargetStrategy = new FindTargetGreedyByUsageInfo(containerManager, placementPolicyValidateProxy, nodeManager); } - this.iterationsStatistic = new ArrayList<>(); + this.iterationsStatistic = new ConcurrentLinkedQueue<>(); } /** @@ -308,43 +309,42 @@ private void balance() { private void saveIterationStatistic(Integer iterationNumber, IterationResult iR) { ContainerBalancerTaskIterationStatusInfo iterationStatistic = new ContainerBalancerTaskIterationStatusInfo( - iterationNumber, - iR.name(), - getSizeScheduledForMoveInLatestIteration() / OzoneConsts.GB, - metrics.getDataSizeMovedGBInLatestIteration(), - metrics.getNumContainerMovesScheduledInLatestIteration(), - metrics.getNumContainerMovesCompletedInLatestIteration(), - metrics.getNumContainerMovesFailedInLatestIteration(), - metrics.getNumContainerMovesTimeoutInLatestIteration(), - findTargetStrategy.getSizeEnteringNodes() - .entrySet() - .stream() - .filter(Objects::nonNull) - .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0) - .collect( - Collectors.toMap( - entry -> entry.getKey().getUuid(), - entry -> entry.getValue() / OzoneConsts.GB - ) - ), - findSourceStrategy.getSizeLeavingNodes() - .entrySet() - .stream() - .filter(Objects::nonNull) - .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0) - .collect( - Collectors.toMap( - entry -> entry.getKey().getUuid(), - entry -> entry.getValue() / OzoneConsts.GB - ) - ) + iterationNumber, + iR.name(), + getSizeScheduledForMoveInLatestIteration() / OzoneConsts.GB, + metrics.getDataSizeMovedGBInLatestIteration(), + metrics.getNumContainerMovesScheduledInLatestIteration(), + metrics.getNumContainerMovesCompletedInLatestIteration(), + metrics.getNumContainerMovesFailedInLatestIteration(), + metrics.getNumContainerMovesTimeoutInLatestIteration(), + findTargetStrategy.getSizeEnteringNodes() + .entrySet() + .stream() + .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0) + .collect( + Collectors.toMap( + entry -> entry.getKey().getUuid(), + entry -> entry.getValue() / OzoneConsts.GB + ) + ), + findSourceStrategy.getSizeLeavingNodes() + .entrySet() + .stream() + .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0) + .collect( + Collectors.toMap( + entry -> entry.getKey().getUuid(), + entry -> entry.getValue() / OzoneConsts.GB + ) + ) ); - iterationsStatistic.add(iterationStatistic); + iterationsStatistic.offer(iterationStatistic); } public List getCurrentIterationsStatistic() { + List resultList = new ArrayList<>(iterationsStatistic); - int lastIterationNumber = iterationsStatistic.stream() + int lastIterationNumber = resultList.stream() .mapToInt(ContainerBalancerTaskIterationStatusInfo::getIterationNumber) .max() .orElse(0); @@ -361,9 +361,9 @@ public List getCurrentIterationsStatis findTargetStrategy.getSizeEnteringNodes() .entrySet() .stream() - .filter(Objects::nonNull) .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0) - .collect(Collectors.toMap( + .collect( + Collectors.toMap( entry -> entry.getKey().getUuid(), entry -> entry.getValue() / OzoneConsts.GB ) @@ -371,7 +371,6 @@ public List getCurrentIterationsStatis findSourceStrategy.getSizeLeavingNodes() .entrySet() .stream() - .filter(Objects::nonNull) .filter(datanodeDetailsLongEntry -> datanodeDetailsLongEntry.getValue() > 0) .collect( Collectors.toMap( @@ -380,7 +379,6 @@ public List getCurrentIterationsStatis ) ) ); - List resultList = new ArrayList<>(iterationsStatistic); resultList.add(currentIterationStatistic); return resultList; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java index 435cc9859a94..57cc8b32b949 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java @@ -26,11 +26,11 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.PriorityQueue; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; /** * The selection criteria for selecting source datanodes , the containers of @@ -46,7 +46,7 @@ public class FindSourceGreedy implements FindSourceStrategy { private Double lowerLimit; FindSourceGreedy(NodeManager nodeManager) { - sizeLeavingNode = new HashMap<>(); + sizeLeavingNode = new ConcurrentHashMap<>(); potentialSources = new PriorityQueue<>((a, b) -> { double currentUsageOfA = a.calculateUtilization( -sizeLeavingNode.get(a.getDatanodeDetails()));