diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java index 96db6d13fea5..ade4cbcab3c4 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java @@ -19,21 +19,27 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableRate; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.util.PerformanceMetrics; +import org.apache.hadoop.util.PerformanceMetricsInitializer; /** * The client metrics for the Storage Container protocol. */ @InterfaceAudience.Private @Metrics(about = "Storage Container Client Metrics", context = "dfs") -public class XceiverClientMetrics { +public class XceiverClientMetrics implements MetricsSource { public static final String SOURCE_NAME = XceiverClientMetrics.class .getSimpleName(); @@ -43,8 +49,11 @@ public class XceiverClientMetrics { private @Metric MutableCounterLong ecReconstructionFailsTotal; private MutableCounterLong[] pendingOpsArray; private MutableCounterLong[] opsArray; - private MutableRate[] containerOpsLatency; + private PerformanceMetrics[] containerOpsLatency; private MetricsRegistry registry; + private OzoneConfiguration conf = new OzoneConfiguration(); + private int[] intervals = conf.getInts(OzoneConfigKeys + .OZONE_XCEIVER_CLIENT_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY); public XceiverClientMetrics() { init(); @@ -56,7 +65,7 @@ public void init() { this.pendingOpsArray = new MutableCounterLong[numEnumEntries]; this.opsArray = new MutableCounterLong[numEnumEntries]; - this.containerOpsLatency = new MutableRate[numEnumEntries]; + this.containerOpsLatency = new PerformanceMetrics[numEnumEntries]; for (int i = 0; i < numEnumEntries; i++) { pendingOpsArray[i] = registry.newCounter( "numPending" + ContainerProtos.Type.forNumber(i + 1), @@ -66,11 +75,11 @@ public void init() { .newCounter("opCount" + ContainerProtos.Type.forNumber(i + 1), "number of" + ContainerProtos.Type.forNumber(i + 1) + " ops", (long) 0); - - containerOpsLatency[i] = registry.newRate( - ContainerProtos.Type.forNumber(i + 1) + "Latency", - "latency of " + ContainerProtos.Type.forNumber(i + 1) - + " ops"); + containerOpsLatency[i] = + PerformanceMetricsInitializer.getMetrics(registry, + ContainerProtos.Type.forNumber(i + 1) + "Latency", + "latency of " + ContainerProtos.Type.forNumber(i + 1), + "Ops", "Time", intervals); } } @@ -129,4 +138,21 @@ public void unRegister() { MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME); } + + @Override + public void getMetrics(MetricsCollector collector, boolean b) { + MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME); + + pendingOps.snapshot(recordBuilder, true); + totalOps.snapshot(recordBuilder, true); + ecReconstructionTotal.snapshot(recordBuilder, true); + ecReconstructionFailsTotal.snapshot(recordBuilder, true); + + int numEnumEntries = ContainerProtos.Type.values().length; + for (int i = 0; i < numEnumEntries; i++) { + pendingOpsArray[i].snapshot(recordBuilder, true); + opsArray[i].snapshot(recordBuilder, true); + containerOpsLatency[i].snapshot(recordBuilder, true); + } + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index c7867ffdcbee..c9a3a4085202 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -672,12 +672,17 @@ public final class OzoneConfigKeys { public static final String HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY = "hdds.scmclient.failover.max.retry"; + + public static final String OZONE_XCEIVER_CLIENT_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY = + "ozone.xceiver.client.metrics.percentiles.intervals.seconds"; + public static final String OZONE_OM_NETWORK_TOPOLOGY_REFRESH_DURATION = "ozone.om.network.topology.refresh.duration"; public static final String OZONE_OM_NETWORK_TOPOLOGY_REFRESH_DURATION_DEFAULT = "1h"; + /** * There is no need to instantiate this class. */ diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java index 83d61cab6857..0a50eab19509 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java @@ -70,7 +70,7 @@ public static void initialize(T source, MetricsRegistry registry, * @param intervals intervals for quantiles * @return an instance of PerformanceMetrics */ - private static PerformanceMetrics getMetrics( + public static PerformanceMetrics getMetrics( MetricsRegistry registry, String name, String description, String sampleName, String valueName, int[] intervals) { return new PerformanceMetrics( diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index fc873f20af69..5a4b9a22c887 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1914,7 +1914,14 @@ Setting this interval equal to the metrics sampling time ensures more detailed metrics. - + + ozone.xceiver.client.metrics.percentiles.intervals.seconds + 60 + XCEIVER, PERFORMANCE + Specifies the interval in seconds for the rollover of XceiverClient MutableQuantiles metrics. + Setting this interval equal to the metrics sampling time ensures more detailed metrics. + + ozone.om.save.metrics.interval 5m