HDDS-10182. Add P99 quantiles and Min/Max Metrics for Xceiver Client Performance Metrics

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java

@@ -19,21 +19,27 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.util.PerformanceMetrics;
+import org.apache.hadoop.util.PerformanceMetricsInitializer;
 
 /**
  * The client metrics for the Storage Container protocol.
  */
 @InterfaceAudience.Private
 @Metrics(about = "Storage Container Client Metrics", context = "dfs")
-public class XceiverClientMetrics {
+public class XceiverClientMetrics implements MetricsSource {
   public static final String SOURCE_NAME = XceiverClientMetrics.class
       .getSimpleName();
 
@@ -43,8 +49,11 @@ public class XceiverClientMetrics {
   private @Metric MutableCounterLong ecReconstructionFailsTotal;
   private MutableCounterLong[] pendingOpsArray;
   private MutableCounterLong[] opsArray;
-  private MutableRate[] containerOpsLatency;
+  private PerformanceMetrics[] containerOpsLatency;
   private MetricsRegistry registry;
+  private OzoneConfiguration conf = new OzoneConfiguration();
+  private int[] intervals = conf.getInts(OzoneConfigKeys
+      .OZONE_XCEIVER_CLIENT_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
 
   public XceiverClientMetrics() {
     init();
@@ -56,7 +65,7 @@ public void init() {
 
     this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
     this.opsArray = new MutableCounterLong[numEnumEntries];
-    this.containerOpsLatency = new MutableRate[numEnumEntries];
+    this.containerOpsLatency = new PerformanceMetrics[numEnumEntries];
     for (int i = 0; i < numEnumEntries; i++) {
       pendingOpsArray[i] = registry.newCounter(
           "numPending" + ContainerProtos.Type.forNumber(i + 1),
@@ -66,11 +75,11 @@ public void init() {
           .newCounter("opCount" + ContainerProtos.Type.forNumber(i + 1),
               "number of" + ContainerProtos.Type.forNumber(i + 1) + " ops",
               (long) 0);
-
-      containerOpsLatency[i] = registry.newRate(
-          ContainerProtos.Type.forNumber(i + 1) + "Latency",
-          "latency of " + ContainerProtos.Type.forNumber(i + 1)
-          + " ops");
+      containerOpsLatency[i] =
+          PerformanceMetricsInitializer.getMetrics(registry,
+              ContainerProtos.Type.forNumber(i + 1) + "Latency",
+              "latency of " + ContainerProtos.Type.forNumber(i + 1),
+              "Ops", "Time", intervals);
     }
   }
 
@@ -129,4 +138,21 @@ public void unRegister() {
     MetricsSystem ms = DefaultMetricsSystem.instance();
     ms.unregisterSource(SOURCE_NAME);
   }
+
+  @Override
+  public void getMetrics(MetricsCollector collector, boolean b) {
+    MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME);
+
+    pendingOps.snapshot(recordBuilder, true);
+    totalOps.snapshot(recordBuilder, true);
+    ecReconstructionTotal.snapshot(recordBuilder, true);
+    ecReconstructionFailsTotal.snapshot(recordBuilder, true);
+
+    int numEnumEntries = ContainerProtos.Type.values().length;
+    for (int i = 0; i < numEnumEntries; i++) {
+      pendingOpsArray[i].snapshot(recordBuilder, true);
+      opsArray[i].snapshot(recordBuilder, true);
+      containerOpsLatency[i].snapshot(recordBuilder, true);
+    }
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java

@@ -672,12 +672,17 @@ public final class OzoneConfigKeys {
   public static final String HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY =
       "hdds.scmclient.failover.max.retry";
 
+
+  public static final String OZONE_XCEIVER_CLIENT_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY =
+      "ozone.xceiver.client.metrics.percentiles.intervals.seconds";
+
   public static final String
       OZONE_OM_NETWORK_TOPOLOGY_REFRESH_DURATION =
       "ozone.om.network.topology.refresh.duration";
   public static final String
       OZONE_OM_NETWORK_TOPOLOGY_REFRESH_DURATION_DEFAULT = "1h";
 
+
   /**
    * There is no need to instantiate this class.
    */

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java

@@ -70,7 +70,7 @@ public static <T> void initialize(T source, MetricsRegistry registry,
    * @param intervals intervals for quantiles
    * @return an instance of PerformanceMetrics
    */
-  private static PerformanceMetrics getMetrics(
+  public static PerformanceMetrics getMetrics(
       MetricsRegistry registry, String name, String description,
       String sampleName, String valueName, int[] intervals) {
     return new PerformanceMetrics(

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1914,7 +1914,14 @@
       Setting this interval equal to the metrics sampling time ensures more detailed metrics.
     </description>
   </property>
-
+  <property>
+    <name>ozone.xceiver.client.metrics.percentiles.intervals.seconds</name>
+    <value>60</value>
+    <tag>XCEIVER, PERFORMANCE</tag>
+    <description>Specifies the interval in seconds for the rollover of XceiverClient MutableQuantiles metrics.
+      Setting this interval equal to the metrics sampling time ensures more detailed metrics.
+    </description>
+  </property>
   <property>
     <name>ozone.om.save.metrics.interval</name>
     <value>5m</value>