Skip to content

HDDS-4929. Select target datanodes and containers to move for Container Balancer #2441

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lokeshj1703 merged 15 commits into from
Aug 11, 2021
Merged

HDDS-4929. Select target datanodes and containers to move for Container Balancer #2441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
bdbfaa1
HDDS-4929. Select target datanodes and containers to move.
siddhantsangwan Jun 10, 2021
fa2314e
HDDS-4929. Resolve problems in ContainerBalancer.
siddhantsangwan Jun 25, 2021
9118688
Merge remote-tracking branch 'apache/master' into HDDS-4929
siddhantsangwan Jun 25, 2021
cde49e4
HDDS-4929. Fix and improve logic for container move selection. Add so…
siddhantsangwan Jul 14, 2021
87bae4f
HDDS-4929. Add more unit tests and docs.
siddhantsangwan Jul 19, 2021
7b79dc2
Merge branch 'master' into HDDS-4929
siddhantsangwan Jul 19, 2021
8c44896
HDDS-4929. Resolve issues.
siddhantsangwan Jul 19, 2021
2f45e5f
HDDS-4929. Fix findbugs issues. Delete unneccessary code in Container…
siddhantsangwan Jul 22, 2021
1d4c495
HDDS-4929. Integrate with ReplicationManager#move.
siddhantsangwan Jul 27, 2021
d5d886a
Merge branch 'master' into HDDS-4929
siddhantsangwan Jul 27, 2021
3c7c5ce
HDDS-4929. Add a balancing interval for waiting before each iteration…
siddhantsangwan Aug 2, 2021
e3deed1
Merge branch 'master' into HDDS-4929
siddhantsangwan Aug 2, 2021
66e6922
HDDS-4929. Check if SCM is leader in ContainerBalancer#initializeIter…
siddhantsangwan Aug 2, 2021
fd3a039
HDDS-4929. Fix issues. Refactor for maxDatanodesRatioToInvolvePerIter…
siddhantsangwan Aug 10, 2021
c8d8bdf
HDDS-4929. Ignore testContainerBalancerCLIOperations. Delete unused c…
siddhantsangwan Aug 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,8 @@ Map<String, Pair<Boolean, String>> getSafeModeRuleStatuses()
*/
boolean startContainerBalancer(Optional<Double> threshold,
Optional<Integer> idleiterations,
Optional<Integer> maxDatanodesToBalance,
Optional<Long> maxSizeToMoveInGB) throws IOException;
Optional<Double> maxDatanodesRatioToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB) throws IOException;

/**
* Stop ContainerBalancer.
Expand Down
11 changes: 5 additions & 6 deletions ...n/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,29 +18,28 @@
package org.apache.hadoop.hdds.scm.protocol;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;

import java.io.Closeable;
import java.io.IOException;
import java.util.EnumSet;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;

/**
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container.
Expand Down Expand Up @@ -294,8 +293,8 @@ Map<String, Pair<Boolean, String>> getSafeModeRuleStatuses()
*/
boolean startContainerBalancer(Optional<Double> threshold,
Optional<Integer> idleiterations,
Optional<Integer> maxDatanodesToBalance,
Optional<Long> maxSizeToMoveInGB) throws IOException;
Optional<Double> maxDatanodesRatioToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB) throws IOException;

/**
* Stop ContainerBalancer.
Expand Down
30 changes: 17 additions & 13 deletions ...he/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -726,10 +726,10 @@ public boolean getReplicationManagerStatus() throws IOException {
}

@Override
public boolean startContainerBalancer(Optional<Double> threshold,
Optional<Integer> idleiterations,
Optional<Integer> maxDatanodesToBalance,
Optional<Long> maxSizeToMoveInGB) throws IOException{
public boolean startContainerBalancer(
Optional<Double> threshold, Optional<Integer> idleiterations,
Optional<Double> maxDatanodesRatioToInvolvePerIteration,
Optional<Long> maxSizeToMovePerIterationInGB) throws IOException{
StartContainerBalancerRequestProto.Builder builder =
StartContainerBalancerRequestProto.newBuilder();
builder.setTraceID(TracingUtil.exportCurrentSpan());
Expand All @@ -741,17 +741,21 @@ public boolean startContainerBalancer(Optional<Double> threshold,
"threshold should to be specified in range [0.0, 1.0).");
builder.setThreshold(tsd);
}
if (maxSizeToMoveInGB.isPresent()) {
long mstm = maxSizeToMoveInGB.get();
if (maxSizeToMovePerIterationInGB.isPresent()) {
long mstm = maxSizeToMovePerIterationInGB.get();
Preconditions.checkState(mstm > 0,
"maxSizeToMoveInGB must be positive.");
builder.setMaxSizeToMoveInGB(mstm);
"maxSizeToMovePerIterationInGB must be positive.");
builder.setMaxSizeToMovePerIterationInGB(mstm);
}
if (maxDatanodesToBalance.isPresent()) {
int mdtb = maxDatanodesToBalance.get();
Preconditions.checkState(mdtb > 0,
"maxDatanodesToBalance must be positive.");
builder.setMaxDatanodesToBalance(mdtb);
if (maxDatanodesRatioToInvolvePerIteration.isPresent()) {
double mdti = maxDatanodesRatioToInvolvePerIteration.get();
Preconditions.checkState(mdti >= 0,
"maxDatanodesRatioToInvolvePerIteration must be " +
"greater than equal to zero.");
Preconditions.checkState(mdti <= 1,
"maxDatanodesRatioToInvolvePerIteration must be " +
"lesser than equal to one.");
builder.setMaxDatanodesRatioToInvolvePerIteration(mdti);
}
if (idleiterations.isPresent()) {
int idi = idleiterations.get();
Expand Down
4 changes: 2 additions & 2 deletions hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,8 +479,8 @@ message StartContainerBalancerRequestProto {
optional string traceID = 1;
optional double threshold = 2;
optional int32 idleiterations = 3;
optional int32 maxDatanodesToBalance = 4;
optional int64 maxSizeToMoveInGB = 5;
optional double maxDatanodesRatioToInvolvePerIteration = 4;
optional int64 maxSizeToMovePerIterationInGB = 5;
}

message StartContainerBalancerResponseProto {
Expand Down
7 changes: 6 additions & 1 deletion ...dds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public class ReplicationManager implements SCMService {
* the corresponding reason. this is useful for tracking
* the result of move option
*/
enum MoveResult {
public enum MoveResult {
// both replication and deletion are completed
COMPLETED,
// RM is not running
Expand Down Expand Up @@ -1571,6 +1571,11 @@ private boolean isOpenContainerHealthy(
.allMatch(r -> ReplicationManager.compareState(state, r.getState()));
}

public boolean isContainerReplicatingOrDeleting(ContainerID containerID) {
return inflightReplication.containsKey(containerID) ||
inflightDeletion.containsKey(containerID);
}

/**
* Wrapper class to hold the InflightAction with its start time.
*/
Expand Down
Loading