HDDS-9717. Add P99 quantiles and Min/Max Metrics for S3G Performance Metrics

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java

@@ -17,12 +17,18 @@
  */
 package org.apache.hadoop.util;
 
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.ratis.util.function.CheckedRunnable;
 import org.apache.ratis.util.function.CheckedSupplier;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 /**
  * Encloses helpers to deal with metrics.
@@ -63,4 +69,34 @@ public static <T, E extends IOException> T captureLatencyNs(
       latencySetter.accept(Time.monotonicNowNanos() - start);
     }
   }
+
+  /**
+   * Creates MutableQuantiles metrics with one or multiple intervals.
+   *
+   * @param registry    The MetricsRegistry to register the new MutableQuantiles
+   *                   instances.
+   * @param name        The base name of the metric.
+   * @param description The description of the metric.
+   * @param sampleName of the metric (e.g., "Ops")
+   * @param valueName of the metric (e.g., "Time" or "Latency")
+   * @param intervals   An array of intervals for the quantiles.
+   * @return A list of created MutableQuantiles instances.
+   */
+  public static List<MutableQuantiles> createQuantiles(MetricsRegistry registry,
+      String name, String description, String sampleName, String valueName,
+      int... intervals) {
+    if (intervals == null) {
+      throw new IllegalArgumentException(
+          "At least one interval should be provided.");
+    }
+    if (intervals.length == 0) {
+      return new ArrayList<>();
+    }
+
+    return Arrays.stream(intervals).mapToObj(interval -> {
+      String quantileName = name + interval + "s";
+      return registry.newQuantiles(quantileName, description,
+          sampleName, valueName, interval);
+    }).collect(Collectors.toList());
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableMetric;
+import org.apache.hadoop.metrics2.util.SampleStat.MinMax;
+import org.apache.commons.lang3.StringUtils;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+/**
+ * A mutable metric that tracks the minimum and maximum
+ * values of a dataset over time.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class MutableMinMax extends MutableMetric {
+  private final MinMax intervalMinMax = new MinMax();
+  private final MinMax prevMinMax = new MinMax();
+  private final MetricsInfo iMinInfo;
+  private final MetricsInfo iMaxInfo;
+
+  /**
+   * Construct a minMax metric.
+   * @param registry    MetricsRegistry of the metric
+   * @param name        of the metric
+   * @param description of the metric
+   * @param valueName   of the metric (e.g. "Time", "Latency")
+   */
+  public MutableMinMax(MetricsRegistry registry,
+      String name, String description, String valueName) {
+    String ucName = StringUtils.capitalize(name);
+    String desc = StringUtils.uncapitalize(description);
+    String uvName = StringUtils.capitalize(valueName);
+    String lvName = StringUtils.uncapitalize(valueName);
+    iMinInfo = info(ucName + "IMin" + uvName,
+        "Min " + lvName + " for " + desc + "in the last reporting interval");
+    iMaxInfo = info(ucName + "IMax" + uvName,
+        "Max " + lvName + " for " + desc + "in the last reporting interval");
+    // hadoop.metrics2 only supports standard types of Metrics registered
+    // with annotations, but not custom types of metrics.
+    // Registering here is for compatibility with metric classes
+    // that are only registered with annotations and do not override getMetrics.
+    registry.newGauge(iMinInfo, 0);
+    registry.newGauge(iMaxInfo, 0);
+  }
+
+  /**
+   * Add a snapshot to the metric.
+   * @param value of the metric
+   */
+  public synchronized void add(long value) {
+    intervalMinMax.add(value);
+    setChanged();
+  }
+
+  private MinMax lastMinMax() {
+    return changed() ? intervalMinMax : prevMinMax;
+  }
+
+  @Override
+  public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) {
+    if (all || this.changed()) {
+      builder.addGauge(iMinInfo, lastMinMax().min());
+      builder.addGauge(iMaxInfo, lastMinMax().max());
+      if (changed()) {
+        prevMinMax.reset(intervalMinMax);
+        intervalMinMax.reset();
+        clearChanged();
+      }
+    }
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.MutableStat;
+
+import java.util.List;
+
+/**
+ * The {@code PerformanceMetrics} class encapsulates a collection of related
+ * metrics including a MutableStat, MutableQuantiles, and a MutableMinMax.
+ * This class provides methods to update these metrics and to
+ * snapshot their values for reporting.
+ */
+public class PerformanceMetrics {
+  private final MutableStat stat;
+  private final List<MutableQuantiles> quantiles;
+  private final MutableMinMax minMax;
+
+  /**
+   * Initializes aggregated metrics for the specified metrics source.
+   *
+   * @param source the metrics source
+   * @param registry the metrics registry
+   * @param intervals the intervals for quantiles computation. Note, each
+   *        interval in 'intervals' increases memory usage, as it corresponds
+   *        to a separate quantile calculator.
+   */
+  public static synchronized <T> void initializeMetrics(T source,
+      MetricsRegistry registry, String sampleName, String valueName,
+      int[] intervals) {
+    try {
+      PerformanceMetricsInitializer.initialize(
+          source, registry, sampleName, valueName, intervals);
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException("Failed to initialize PerformanceMetrics", e);
+    }
+  }
+
+  /**
+   * Construct an instance of PerformanceMetrics with the specified MutableStat,
+   * MutableQuantiles, and MutableMinMax.
+   *
+   * @param stat the stat metric
+   * @param quantiles the quantiles metrics
+   * @param minMax the min/max tracker
+   */
+  public PerformanceMetrics(MutableStat stat,
+      List<MutableQuantiles> quantiles, MutableMinMax minMax) {
+    this.stat = stat;
+    this.quantiles = quantiles;
+    this.minMax = minMax;
+  }
+
+  /**
+   * Adds a value to all the aggregated metrics.
+   *
+   * @param value the value to add
+   */
+  public void add(long value) {
+    this.stat.add(value);
+    this.quantiles.forEach(quantile -> quantile.add(value));
+    this.minMax.add(value);
+  }
+
+  /**
+   * Snapshots the values of all the aggregated metrics for reporting.
+   *
+   * @param recordBuilder the metrics record builder
+   * @param all flag to indicate whether to snapshot all metrics or only changed
+   */
+  public void snapshot(MetricsRecordBuilder recordBuilder, boolean all) {
+    this.stat.snapshot(recordBuilder, all);
+    this.quantiles.forEach(quantile -> quantile.snapshot(recordBuilder, all));
+    this.minMax.snapshot(recordBuilder, all);
+  }
+
+}
+

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java

@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+
+import java.lang.reflect.Field;
+
+/**
+ * Utility class for initializing PerformanceMetrics in a MetricsSource.
+ */
+public final class PerformanceMetricsInitializer {
+  private PerformanceMetricsInitializer() { }
+
+  /**
+   * Initializes aggregated metrics in the given metrics source.
+   *
+   * @param source the metrics source
+   * @param registry the metrics registry
+   * @param sampleName sample name
+   * @param valueName value name
+   * @param intervals intervals for quantiles
+   * @throws IllegalAccessException if unable to access the field
+   */
+  public static <T> void initialize(T source, MetricsRegistry registry,
+      String sampleName, String valueName, int[] intervals)
+      throws IllegalAccessException {
+    Field[] fields = source.getClass().getDeclaredFields();
+
+    for (Field field : fields) {
+      if (field.getType() == PerformanceMetrics.class) {
+        Metric annotation = field.getAnnotation(Metric.class);
+        if (annotation != null) {
+          String description = annotation.about();
+          String name = field.getName();
+          PerformanceMetrics performanceMetrics =
+              getMetrics(registry, name, description,
+                  sampleName, valueName, intervals);
+          field.setAccessible(true);
+          field.set(source, performanceMetrics);
+        }
+      }
+    }
+  }
+
+  /**
+   * Helper method to create PerformanceMetrics.
+   *
+   * @param registry the metrics registry
+   * @param name metric name
+   * @param description metric description
+   * @param sampleName sample name
+   * @param valueName value name
+   * @param intervals intervals for quantiles
+   * @return an instance of PerformanceMetrics
+   */
+  private static PerformanceMetrics getMetrics(
+      MetricsRegistry registry, String name, String description,
+      String sampleName, String valueName, int[] intervals) {
+    return new PerformanceMetrics(
+        registry.newStat(
+            name, description, sampleName, valueName, false),
+        MetricUtil.createQuantiles(
+            registry, name, description, sampleName, valueName, intervals),
+        new MutableMinMax(registry, name, description, valueName));
+  }
+}

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1876,6 +1876,14 @@
       will be used for http authentication.
     </description>
   </property>
+  <property>
+    <name>ozone.s3g.metrics.percentiles.intervals.seconds</name>
+    <value>60</value>
+    <tag>S3GATEWAY, PERFORMANCE</tag>
+    <description>Specifies the interval in seconds for the rollover of MutableQuantiles metrics.
+      Setting this interval equal to the metrics sampling time ensures more detailed metrics.
+    </description>
+  </property>
 
   <property>
     <name>ozone.om.save.metrics.interval</name>

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMultipartObjectGet.java

@@ -27,6 +27,7 @@
 import org.apache.hadoop.ozone.s3.endpoint.MultipartUploadInitiateResponse;
 import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
 import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -101,6 +102,7 @@ public static void init() throws Exception {
     REST.setClient(client);
     REST.setOzoneConfiguration(conf);
     REST.setContext(context);
+    S3GatewayMetrics.create(conf);
   }
 
   private static void startCluster()

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java

@@ -79,7 +79,7 @@ public Void call() throws Exception {
     loginS3GUser(ozoneConfiguration);
     setHttpBaseDir(ozoneConfiguration);
     httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway");
-    metrics = S3GatewayMetrics.create();
+    metrics = S3GatewayMetrics.create(ozoneConfiguration);
     start();
 
     ShutdownHookManager.get().addShutdownHook(() -> {

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java

@@ -81,6 +81,9 @@ public final class S3GatewayConfigKeys {
   public static final boolean OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED_DEFAULT =
       true;
 
+  public static final String OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY
+      = "ozone.s3g.metrics.percentiles.intervals.seconds";
+
   /**
    * Never constructed.
    */

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -387,7 +387,7 @@ protected ClientProtocol getClientProtocol() {
 
   @VisibleForTesting
   public S3GatewayMetrics getMetrics() {
-    return S3GatewayMetrics.create();
+    return S3GatewayMetrics.getMetrics();
   }
 
   protected Map<String, String> getAuditParameters() {