Skip to content

HDDS-8801. ReplicationManager: Add metric to count how often replication is throttled #4864

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 3 commits into from
Jun 9, 2023
Merged

HDDS-8801. ReplicationManager: Add metric to count how often replication is throttled #4864

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fff5758
Increment a metric each time an overloaded exception is thrown
Jun 9, 2023
da4f9a1
Add additionl tests
Jun 9, 2023
8a75e9f
Suppress findbugs and all the metrics inside the metrics class are th…
Jun 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ...cm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.container.replication;

import com.google.common.annotations.VisibleForTesting;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.Config;
Expand Down Expand Up @@ -502,6 +503,7 @@ public void sendThrottledDeleteCommand(final ContainerInfo container,
int commandCount = nodeManager.getTotalDatanodeCommandCount(datanode,
Type.deleteContainerCommand);
if (commandCount >= datanodeDeleteLimit) {
metrics.incrDeleteContainerCmdsDeferredTotal();
throw new CommandTargetOverloadedException("Cannot schedule a delete " +
"container command for container " + container.containerID() +
" on datanode " + datanode + " as it has too many pending delete " +
Expand Down Expand Up @@ -533,6 +535,7 @@ public void sendThrottledReplicationCommand(ContainerInfo containerInfo,
List<Pair<Integer, DatanodeDetails>> sourceWithCmds =
getAvailableDatanodesForReplication(sources);
if (sourceWithCmds.isEmpty()) {
metrics.incrReplicateContainerCmdsDeferredTotal();
throw new CommandTargetOverloadedException("No sources with capacity " +
"available for replication of container " + containerID + " to " +
target);
Expand All @@ -553,6 +556,7 @@ public void sendThrottledReconstructionCommand(ContainerInfo containerInfo,
List<Pair<Integer, DatanodeDetails>> targetWithCmds =
getAvailableDatanodesForReplication(targets);
if (targetWithCmds.isEmpty()) {
metrics.incrECReconstructionCmdsDeferredTotal();
throw new CommandTargetOverloadedException("No target with capacity " +
"available for reconstruction of " + containerInfo.getContainerID());
}
Expand Down Expand Up @@ -1407,6 +1411,7 @@ public String getServiceName() {
return ReplicationManager.class.getSimpleName();
}

@SuppressFBWarnings("IS2_INCONSISTENT_SYNC")
public ReplicationManagerMetrics getMetrics() {
return metrics;
}
Expand Down
41 changes: 41 additions & 0 deletions ...main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,19 @@ public final class ReplicationManagerMetrics implements MetricsSource {
@Metric("Number of EC replicas scheduled for delete which timed out.")
private MutableCounterLong ecReplicaDeleteTimeoutTotal;

@Metric("NUmber of Reconstruct EC Container commands that could not be sent "
+ "due to the pending commands on the target datanode")
private MutableCounterLong ecReconstructionCmdsDeferredTotal;

@Metric("Number of delete container commands that could not be sent due "
+ "to the pending commands on the target datanode")
private MutableCounterLong deleteContainerCmdsDeferredTotal;

@Metric("Number of replicate container commands that could not be sent due "
+ "to the pending commands on all source datanodes")
private MutableCounterLong replicateContainerCmdsDeferredTotal;


public ReplicationManagerMetrics(ReplicationManager manager) {
this.registry = new MetricsRegistry(METRICS_SOURCE_NAME);
this.replicationManager = manager;
Expand Down Expand Up @@ -223,6 +236,9 @@ public void getMetrics(MetricsCollector collector, boolean all) {
ecReconstructionCmdsSentTotal.snapshot(builder, all);
ecReplicaCreateTimeoutTotal.snapshot(builder, all);
ecReplicasDeletedTotal.snapshot(builder, all);
ecReconstructionCmdsDeferredTotal.snapshot(builder, all);
deleteContainerCmdsDeferredTotal.snapshot(builder, all);
replicateContainerCmdsDeferredTotal.snapshot(builder, all);
}

public void unRegister() {
Expand Down Expand Up @@ -372,6 +388,18 @@ public void incrEcReconstructionCmdsSentTotal() {
this.ecReconstructionCmdsSentTotal.incr();
}

public void incrECReconstructionCmdsDeferredTotal() {
this.ecReconstructionCmdsDeferredTotal.incr();
}

public void incrDeleteContainerCmdsDeferredTotal() {
this.deleteContainerCmdsDeferredTotal.incr();
}

public void incrReplicateContainerCmdsDeferredTotal() {
this.replicateContainerCmdsDeferredTotal.incr();
}

public long getEcReplication() {
return replicationManager.getContainerReplicaPendingOps()
.getPendingOpCount(ContainerReplicaOp.PendingOpType.ADD);
Expand Down Expand Up @@ -417,4 +445,17 @@ public long getEcReplicasCreatedTotal() {
public long getEcReplicasDeletedTotal() {
return ecReplicasDeletedTotal.value();
}

public long getEcReconstructionCmdsDeferredTotal() {
return ecReconstructionCmdsDeferredTotal.value();
}

public long getDeleteContainerCmdsDeferredTotal() {
return deleteContainerCmdsDeferredTotal.value();
}

public long getReplicateContainerCmdsDeferredTotal() {
return replicateContainerCmdsDeferredTotal.value();
}

}
Loading