From 804c0168397e461501273b514770be2433e2f5bc Mon Sep 17 00:00:00 2001
From: "pony.chen" <pony.chen@shopee.com>
Date: Fri, 17 Nov 2023 20:19:32 +0800
Subject: [PATCH 01/12] Add P99 quantiles and Min/Max Metrics for S3G
 Performance Metrics

---
 .../apache/hadoop/util/AggregatedMetrics.java | 95 +++++++++++++++++++
 .../util/AggregatedMetricsInitializer.java    | 83 ++++++++++++++++
 .../org/apache/hadoop/util/MetricUtil.java    | 36 +++++++
 .../org/apache/hadoop/util/MutableMinMax.java | 93 ++++++++++++++++++
 .../org/apache/hadoop/ozone/s3/Gateway.java   |  2 +-
 .../hadoop/ozone/s3/S3GatewayConfigKeys.java  |  3 +
 .../ozone/s3/endpoint/EndpointBase.java       |  2 +-
 .../s3/endpoint/ObjectEndpointStreaming.java  |  2 +-
 .../ozone/s3/metrics/S3GatewayMetrics.java    | 94 ++++++++++--------
 9 files changed, 365 insertions(+), 45 deletions(-)
 create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java
 create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java
 create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java
new file mode 100644
index 000000000000..1177aa683ab0
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.MutableStat;
+
+import java.util.List;
+
+/**
+ * The {@code AggregatedMetrics} class encapsulates a collection of related
+ * metrics including a MutableStat, MutableQuantiles, and a MutableMinMax.
+ * This class provides methods to update these metrics and to
+ * snapshot their values for reporting.
+ */
+public class AggregatedMetrics {
+  private final MutableStat stat;
+  private final List<MutableQuantiles> quantiles;
+  private final MutableMinMax minMax;
+
+  /**
+   * Initializes aggregated metrics for the specified metrics source.
+   *
+   * @param source the metrics source
+   * @param registry the metrics registry
+   * @param intervals the intervals for quantiles computation
+   */
+  public static synchronized <T> void initializeMetrics(T source,
+      MetricsRegistry registry, String sampleName, String valueName,
+      int[] intervals) {
+    try {
+      AggregatedMetricsInitializer.initialize(
+          source, registry, sampleName, valueName, intervals);
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException("Failed to initialize AggregatedMetrics", e);
+    }
+  }
+
+  /**
+   * Constructs an instance of AggregatedMetrics with the specified MutableStat,
+   * MutableQuantiles, and MutableMinMax.
+   *
+   * @param stat the stat metric
+   * @param quantiles the quantiles metrics
+   * @param minMax the min/max tracker
+   */
+  public AggregatedMetrics(MutableStat stat,
+      List<MutableQuantiles> quantiles, MutableMinMax minMax) {
+    this.stat = stat;
+    this.quantiles = quantiles;
+    this.minMax = minMax;
+  }
+
+  /**
+   * Adds a value to all the aggregated metrics.
+   *
+   * @param value the value to add
+   */
+  public void add(long value) {
+    this.stat.add(value);
+    this.quantiles.forEach(quantile -> quantile.add(value));
+    this.minMax.add(value);
+  }
+
+  /**
+   * Snapshots the values of all the aggregated metrics for reporting.
+   *
+   * @param recordBuilder the metrics record builder
+   * @param all flag to indicate whether to snapshot all metrics or only changed
+   */
+  public void snapshot(MetricsRecordBuilder recordBuilder, boolean all) {
+    this.stat.snapshot(recordBuilder, all);
+    this.quantiles.forEach(quantile -> quantile.snapshot(recordBuilder, all));
+    this.minMax.snapshot(recordBuilder, all);
+  }
+
+}
+
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java
new file mode 100644
index 000000000000..a165885f15d1
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+
+import java.lang.reflect.Field;
+
+/**
+ * Utility class for initializing AggregatedMetrics in a MetricsSource.
+ */
+public final class AggregatedMetricsInitializer {
+  private AggregatedMetricsInitializer() { }
+
+  /**
+   * Initializes aggregated metrics in the given metrics source.
+   *
+   * @param source the metrics source
+   * @param registry the metrics registry
+   * @param sampleName sample name
+   * @param valueName value name
+   * @param intervals intervals for quantiles
+   * @throws IllegalAccessException if unable to access the field
+   */
+  public static <T> void initialize(T source, MetricsRegistry registry,
+      String sampleName, String valueName, int[] intervals)
+      throws IllegalAccessException {
+    Field[] fields = source.getClass().getDeclaredFields();
+
+    for (Field field : fields) {
+      if (field.getType() == AggregatedMetrics.class) {
+        Metric annotation = field.getAnnotation(Metric.class);
+        if (annotation != null) {
+          String description = annotation.about();
+          String name = field.getName();
+          AggregatedMetrics aggregatedMetrics =
+              getAggregatedMetrics(registry, name, description,
+                  sampleName, valueName, intervals);
+          field.setAccessible(true);
+          field.set(source, aggregatedMetrics);
+        }
+      }
+    }
+  }
+
+  /**
+   * Helper method to create AggregatedMetrics.
+   *
+   * @param registry the metrics registry
+   * @param name metric name
+   * @param description metric description
+   * @param sampleName sample name
+   * @param valueName value name
+   * @param intervals intervals for quantiles
+   * @return an instance of AggregatedMetrics
+   */
+  private static AggregatedMetrics getAggregatedMetrics(
+      MetricsRegistry registry, String name, String description,
+      String sampleName, String valueName, int[] intervals) {
+    return new AggregatedMetrics(
+        registry.newStat(
+            name, description, sampleName, valueName, false),
+        MetricUtil.createQuantiles(
+            registry, name, description, sampleName, valueName, intervals),
+        new MutableMinMax(registry, name, description, valueName));
+  }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java
index 879f8ed0cc1b..381314e69cff 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java
@@ -17,12 +17,18 @@
  */
 package org.apache.hadoop.util;
 
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.ratis.util.function.CheckedRunnable;
 import org.apache.ratis.util.function.CheckedSupplier;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 /**
  * Encloses helpers to deal with metrics.
@@ -63,4 +69,34 @@ public static <T, E extends IOException> T captureLatencyNs(
       latencySetter.accept(Time.monotonicNowNanos() - start);
     }
   }
+
+  /**
+   * Creates MutableQuantiles metrics with one or multiple intervals.
+   *
+   * @param registry    The MetricsRegistry to register the new MutableQuantiles
+   *                   instances.
+   * @param name        The base name of the metric.
+   * @param description The description of the metric.
+   * @param sampleName of the metric (e.g., "Ops")
+   * @param valueName of the metric (e.g., "Time" or "Latency")
+   * @param intervals   An array of intervals for the quantiles.
+   * @return A list of created MutableQuantiles instances.
+   */
+  public static List<MutableQuantiles> createQuantiles(MetricsRegistry registry,
+      String name, String description, String sampleName, String valueName,
+      int... intervals) {
+    if (intervals == null) {
+      throw new IllegalArgumentException(
+          "At least one interval should be provided.");
+    }
+    if (intervals.length == 0) {
+      return new ArrayList<>();
+    }
+
+    return Arrays.stream(intervals).mapToObj(interval -> {
+      String quantileName = name + interval + "s";
+      return registry.newQuantiles(quantileName, description,
+          sampleName, valueName, interval);
+    }).collect(Collectors.toList());
+  }
 }
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java
new file mode 100644
index 000000000000..b7d0d284ec3c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableMetric;
+import org.apache.hadoop.metrics2.util.SampleStat.MinMax;
+import org.apache.commons.lang3.StringUtils;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+/**
+ * A mutable metric that tracks the minimum and maximum
+ * values of a dataset over time.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class MutableMinMax extends MutableMetric {
+  private final MinMax intervalMinMax = new MinMax();
+  private final MinMax prevMinMax = new MinMax();
+  private final MetricsInfo iMinInfo;
+  private final MetricsInfo iMaxInfo;
+
+  /**
+   * Construct a minMax metric.
+   * @param registry    MetricsRegistry of the metric
+   * @param name        of the metric
+   * @param description of the metric
+   * @param valueName   of the metric (e.g. "Time", "Latency")
+   */
+  public MutableMinMax(MetricsRegistry registry,
+      String name, String description, String valueName) {
+    String ucName = StringUtils.capitalize(name);
+    String desc = StringUtils.uncapitalize(description);
+    String uvName = StringUtils.capitalize(valueName);
+    String lvName = StringUtils.uncapitalize(valueName);
+    iMinInfo = info(ucName + "IMin" + uvName,
+        "Min " + lvName + " for " + desc + "in the last reporting interval");
+    iMaxInfo = info(ucName + "IMax" + uvName,
+        "Max " + lvName + " for " + desc + "in the last reporting interval");
+    // hadoop.metrics2 only supports standard types of Metrics registered
+    // with annotations, but not custom types of metrics.
+    // Registering here is for compatibility with metric classes
+    // that are only registered with annotations and do not override getMetrics.
+    registry.newGauge(iMinInfo, 0);
+    registry.newGauge(iMaxInfo, 0);
+  }
+
+  /**
+   * Add a snapshot to the metric.
+   * @param value of the metric
+   */
+  public synchronized void add(long value) {
+    intervalMinMax.add(value);
+    setChanged();
+  }
+
+  private MinMax lastMinMax() {
+    return changed() ? intervalMinMax : prevMinMax;
+  }
+
+  @Override
+  public void snapshot(MetricsRecordBuilder builder, boolean all) {
+    if (all || this.changed()) {
+      builder.addGauge(iMinInfo, lastMinMax().min());
+      builder.addGauge(iMaxInfo, lastMinMax().max());
+      if (changed()) {
+        prevMinMax.reset(intervalMinMax);
+        intervalMinMax.reset();
+        clearChanged();
+      }
+    }
+  }
+}
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
index 02bf8ff1f422..fa2024f07fb6 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
@@ -103,7 +103,7 @@ public Void call() throws Exception {
     loginS3GUser(ozoneConfiguration);
     setHttpBaseDir();
     httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway");
-    metrics = S3GatewayMetrics.create();
+    metrics = S3GatewayMetrics.create(ozoneConfiguration);
     start();
 
     ShutdownHookManager.get().addShutdownHook(() -> {
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
index 179c5eeee793..a058e413b963 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
@@ -81,6 +81,9 @@ public final class S3GatewayConfigKeys {
   public static final boolean OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED_DEFAULT =
       true;
 
+  public static final String OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY
+      = "ozone.s3g.metrics.percentiles.intervals.seconds";
+
   /**
    * Never constructed.
    */
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 05b7a62c0623..9b3ef9a365d3 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -378,7 +378,7 @@ protected ClientProtocol getClientProtocol() {
 
   @VisibleForTesting
   public S3GatewayMetrics getMetrics() {
-    return S3GatewayMetrics.create();
+    return S3GatewayMetrics.getMetrics();
   }
 
   protected Map<String, String> getAuditParameters() {
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index b536b3248b8e..908a00736bb8 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -147,7 +147,7 @@ public static Response createMultipartKey(OzoneBucket ozoneBucket, String key,
                                             DigestInputStream body)
       throws IOException, OS3Exception {
     String eTag;
-    S3GatewayMetrics metrics = S3GatewayMetrics.create();
+    S3GatewayMetrics metrics = S3GatewayMetrics.getMetrics();
     // OmMultipartCommitUploadPartInfo can only be gotten after the
     // OzoneDataStreamOutput is closed, so we need to save the
     // KeyDataStreamOutput in the OzoneDataStreamOutput and use it to get the
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index b18b9f3354da..76c766c82376 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.ozone.s3.metrics;
 
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.metrics2.MetricsCollector;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.MetricsSource;
@@ -27,8 +28,9 @@
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
+import org.apache.hadoop.util.AggregatedMetrics;
 import org.apache.hadoop.util.Time;
 
 /**
@@ -95,156 +97,160 @@ public final class S3GatewayMetrics implements MetricsSource {
 
   @Metric(about = "Latency for successfully retrieving an S3 bucket in " +
       "nanoseconds")
-  private MutableRate getBucketSuccessLatencyNs;
+  private AggregatedMetrics getBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve an S3 bucket in nanoseconds")
-  private MutableRate getBucketFailureLatencyNs;
+  private AggregatedMetrics getBucketFailureLatencyNs;
 
   @Metric(about = "Latency for successfully creating an S3 bucket in " +
       "nanoseconds")
-  private MutableRate createBucketSuccessLatencyNs;
+  private AggregatedMetrics createBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to create an S3 bucket in nanoseconds")
-  private MutableRate createBucketFailureLatencyNs;
+  private AggregatedMetrics createBucketFailureLatencyNs;
 
   @Metric(about = "Latency for successfully checking the existence of an " +
       "S3 bucket in nanoseconds")
-  private MutableRate headBucketSuccessLatencyNs;
+  private AggregatedMetrics headBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for successfully deleting an S3 bucket in " +
       "nanoseconds")
-  private MutableRate deleteBucketSuccessLatencyNs;
+  private AggregatedMetrics deleteBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to delete an S3 bucket in nanoseconds")
-  private MutableRate deleteBucketFailureLatencyNs;
+  private AggregatedMetrics deleteBucketFailureLatencyNs;
 
   @Metric(about = "Latency for successfully retrieving an S3 bucket ACL " +
       "in nanoseconds")
-  private MutableRate getAclSuccessLatencyNs;
+  private AggregatedMetrics getAclSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve an S3 bucket ACL " +
       "in nanoseconds")
-  private MutableRate getAclFailureLatencyNs;
+  private AggregatedMetrics getAclFailureLatencyNs;
 
   @Metric(about = "Latency for successfully setting an S3 bucket ACL " +
       "in nanoseconds")
-  private MutableRate putAclSuccessLatencyNs;
+  private AggregatedMetrics putAclSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to set an S3 bucket ACL " +
       "in nanoseconds")
-  private MutableRate putAclFailureLatencyNs;
+  private AggregatedMetrics putAclFailureLatencyNs;
 
   @Metric(about = "Latency for successfully listing multipart uploads " +
       "in nanoseconds")
-  private MutableRate listMultipartUploadsSuccessLatencyNs;
+  private AggregatedMetrics listMultipartUploadsSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to list multipart uploads " +
       "in nanoseconds")
-  private MutableRate listMultipartUploadsFailureLatencyNs;
+  private AggregatedMetrics listMultipartUploadsFailureLatencyNs;
 
   // RootEndpoint
 
   @Metric(about = "Latency for successfully listing S3 buckets " +
       "in nanoseconds")
-  private MutableRate listS3BucketsSuccessLatencyNs;
+  private AggregatedMetrics listS3BucketsSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to list S3 buckets " +
       "in nanoseconds")
-  private MutableRate listS3BucketsFailureLatencyNs;
+  private AggregatedMetrics listS3BucketsFailureLatencyNs;
 
   // ObjectEndpoint
 
   @Metric(about = "Latency for successfully creating a multipart object key " +
       "in nanoseconds")
-  private MutableRate createMultipartKeySuccessLatencyNs;
+  private AggregatedMetrics createMultipartKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to create a multipart object key in " +
       "nanoseconds")
-  private MutableRate createMultipartKeyFailureLatencyNs;
+  private AggregatedMetrics createMultipartKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully copying an S3 object in " +
       "nanoseconds")
-  private MutableRate copyObjectSuccessLatencyNs;
+  private AggregatedMetrics copyObjectSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to copy an S3 object in nanoseconds")
-  private MutableRate copyObjectFailureLatencyNs;
+  private AggregatedMetrics copyObjectFailureLatencyNs;
 
   @Metric(about = "Latency for successfully creating an S3 object key in " +
       "nanoseconds")
-  private MutableRate createKeySuccessLatencyNs;
+  private AggregatedMetrics createKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to create an S3 object key in " +
       "nanoseconds")
-  private MutableRate createKeyFailureLatencyNs;
+  private AggregatedMetrics createKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully listing parts of a multipart " +
       "upload in nanoseconds")
-  private MutableRate listPartsSuccessLatencyNs;
+  private AggregatedMetrics listPartsSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to list parts of a multipart upload " +
       "in nanoseconds")
-  private MutableRate listPartsFailureLatencyNs;
+  private AggregatedMetrics listPartsFailureLatencyNs;
 
   @Metric(about = "Latency for successfully retrieving an S3 object in " +
       "nanoseconds")
-  private MutableRate getKeySuccessLatencyNs;
+  private AggregatedMetrics getKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve an S3 object in nanoseconds")
-  private MutableRate getKeyFailureLatencyNs;
+  private AggregatedMetrics getKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully retrieving metadata for an S3 " +
       "object in nanoseconds")
-  private MutableRate headKeySuccessLatencyNs;
+  private AggregatedMetrics headKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve metadata for an S3 object " +
       "in nanoseconds")
-  private MutableRate headKeyFailureLatencyNs;
+  private AggregatedMetrics headKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully initiating a multipart upload in " +
       "nanoseconds")
-  private MutableRate initMultipartUploadSuccessLatencyNs;
+  private AggregatedMetrics initMultipartUploadSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to initiate a multipart upload in " +
       "nanoseconds")
-  private MutableRate initMultipartUploadFailureLatencyNs;
+  private AggregatedMetrics initMultipartUploadFailureLatencyNs;
 
   @Metric(about = "Latency for successfully completing a multipart upload in " +
       "nanoseconds")
-  private MutableRate completeMultipartUploadSuccessLatencyNs;
+  private AggregatedMetrics completeMultipartUploadSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to complete a multipart upload in " +
       "nanoseconds")
-  private MutableRate completeMultipartUploadFailureLatencyNs;
+  private AggregatedMetrics completeMultipartUploadFailureLatencyNs;
 
   @Metric(about = "Latency for successfully aborting a multipart upload in " +
       "nanoseconds")
-  private MutableRate abortMultipartUploadSuccessLatencyNs;
+  private AggregatedMetrics abortMultipartUploadSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to abort a multipart upload in " +
       "nanoseconds")
-  private MutableRate abortMultipartUploadFailureLatencyNs;
+  private AggregatedMetrics abortMultipartUploadFailureLatencyNs;
 
   @Metric(about = "Latency for successfully deleting an S3 object in " +
       "nanoseconds")
-  private MutableRate deleteKeySuccessLatencyNs;
+  private AggregatedMetrics deleteKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to delete an S3 object in nanoseconds")
-  private MutableRate deleteKeyFailureLatencyNs;
+  private AggregatedMetrics deleteKeyFailureLatencyNs;
 
   @Metric(about = "Latency for put metadata of an key in nanoseconds")
-  private MutableRate putKeyMetadataLatencyNs;
+  private AggregatedMetrics putKeyMetadataLatencyNs;
 
   @Metric(about = "Latency for get metadata of an key in nanoseconds")
-  private MutableRate getKeyMetadataLatencyNs;
+  private AggregatedMetrics getKeyMetadataLatencyNs;
 
   @Metric(about = "Latency for copy metadata of an key in nanoseconds")
-  private MutableRate copyKeyMetadataLatencyNs;
+  private AggregatedMetrics copyKeyMetadataLatencyNs;
 
   /**
    * Private constructor.
    */
-  private S3GatewayMetrics() {
+  private S3GatewayMetrics(OzoneConfiguration conf) {
     this.registry = new MetricsRegistry(SOURCE_NAME);
+    int[] intervals = conf.getInts(S3GatewayConfigKeys
+        .OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
+    AggregatedMetrics.initializeMetrics(
+        this, registry, "Ops", "Time", intervals);
   }
 
   /**
@@ -252,13 +258,13 @@ private S3GatewayMetrics() {
    *
    * @return S3GatewayMetrics
    */
-  public static synchronized S3GatewayMetrics create() {
+  public static synchronized S3GatewayMetrics create(OzoneConfiguration conf) {
     if (instance != null) {
       return instance;
     }
     MetricsSystem ms = DefaultMetricsSystem.instance();
     instance = ms.register(SOURCE_NAME, "S3 Gateway Metrics",
-        new S3GatewayMetrics());
+        new S3GatewayMetrics(conf));
     return instance;
   }
 
@@ -719,4 +725,8 @@ public long getHeadKeyFailure() {
   public long getListS3BucketsFailure() {
     return listS3BucketsFailure.value();
   }
+
+  public static synchronized S3GatewayMetrics getMetrics() {
+    return instance;
+  }
 }

From 62d2db0eaf15c17656a9a23807d440a5fb1299ef Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Mon, 20 Nov 2023 11:42:20 +0800
Subject: [PATCH 02/12] Fix Test

---
 .../java/org/apache/hadoop/ozone/client/OzoneClientStub.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java
index 64f515060b49..2ca712ae9416 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java
@@ -19,6 +19,9 @@
  */
 package org.apache.hadoop.ozone.client;
 
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
+
 /**
  * In-memory OzoneClient for testing.
  */
@@ -29,6 +32,7 @@ public OzoneClientStub() {
 
   public OzoneClientStub(ObjectStoreStub objectStoreStub) {
     super(objectStoreStub, new ClientProtocolStub(objectStoreStub));
+    S3GatewayMetrics.create(new OzoneConfiguration());
   }
 
   @Override

From eb339a63cd0c4ea397326e8273d7aee6bf97c513 Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Mon, 20 Nov 2023 11:55:41 +0800
Subject: [PATCH 03/12] rename AggregatedMetrics to PerformanceMetrics

---
 ...edMetrics.java => PerformanceMetrics.java} | 12 +--
 ...ava => PerformanceMetricsInitializer.java} | 22 ++---
 .../ozone/s3/metrics/S3GatewayMetrics.java    | 80 +++++++++----------
 3 files changed, 57 insertions(+), 57 deletions(-)
 rename hadoop-hdds/common/src/main/java/org/apache/hadoop/util/{AggregatedMetrics.java => PerformanceMetrics.java} (88%)
 rename hadoop-hdds/common/src/main/java/org/apache/hadoop/util/{AggregatedMetricsInitializer.java => PerformanceMetricsInitializer.java} (81%)

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java
similarity index 88%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java
index 1177aa683ab0..4cc645f2a53c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetrics.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java
@@ -25,12 +25,12 @@
 import java.util.List;
 
 /**
- * The {@code AggregatedMetrics} class encapsulates a collection of related
+ * The {@code PerformanceMetrics} class encapsulates a collection of related
  * metrics including a MutableStat, MutableQuantiles, and a MutableMinMax.
  * This class provides methods to update these metrics and to
  * snapshot their values for reporting.
  */
-public class AggregatedMetrics {
+public class PerformanceMetrics {
   private final MutableStat stat;
   private final List<MutableQuantiles> quantiles;
   private final MutableMinMax minMax;
@@ -46,22 +46,22 @@ public static synchronized <T> void initializeMetrics(T source,
       MetricsRegistry registry, String sampleName, String valueName,
       int[] intervals) {
     try {
-      AggregatedMetricsInitializer.initialize(
+      PerformanceMetricsInitializer.initialize(
           source, registry, sampleName, valueName, intervals);
     } catch (IllegalAccessException e) {
-      throw new RuntimeException("Failed to initialize AggregatedMetrics", e);
+      throw new RuntimeException("Failed to initialize PerformanceMetrics", e);
     }
   }
 
   /**
-   * Constructs an instance of AggregatedMetrics with the specified MutableStat,
+   * Constructs an instance of PerformanceMetrics with the specified MutableStat,
    * MutableQuantiles, and MutableMinMax.
    *
    * @param stat the stat metric
    * @param quantiles the quantiles metrics
    * @param minMax the min/max tracker
    */
-  public AggregatedMetrics(MutableStat stat,
+  public PerformanceMetrics(MutableStat stat,
       List<MutableQuantiles> quantiles, MutableMinMax minMax) {
     this.stat = stat;
     this.quantiles = quantiles;
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java
similarity index 81%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java
index a165885f15d1..83d61cab6857 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/AggregatedMetricsInitializer.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetricsInitializer.java
@@ -23,10 +23,10 @@
 import java.lang.reflect.Field;
 
 /**
- * Utility class for initializing AggregatedMetrics in a MetricsSource.
+ * Utility class for initializing PerformanceMetrics in a MetricsSource.
  */
-public final class AggregatedMetricsInitializer {
-  private AggregatedMetricsInitializer() { }
+public final class PerformanceMetricsInitializer {
+  private PerformanceMetricsInitializer() { }
 
   /**
    * Initializes aggregated metrics in the given metrics source.
@@ -44,23 +44,23 @@ public static <T> void initialize(T source, MetricsRegistry registry,
     Field[] fields = source.getClass().getDeclaredFields();
 
     for (Field field : fields) {
-      if (field.getType() == AggregatedMetrics.class) {
+      if (field.getType() == PerformanceMetrics.class) {
         Metric annotation = field.getAnnotation(Metric.class);
         if (annotation != null) {
           String description = annotation.about();
           String name = field.getName();
-          AggregatedMetrics aggregatedMetrics =
-              getAggregatedMetrics(registry, name, description,
+          PerformanceMetrics performanceMetrics =
+              getMetrics(registry, name, description,
                   sampleName, valueName, intervals);
           field.setAccessible(true);
-          field.set(source, aggregatedMetrics);
+          field.set(source, performanceMetrics);
         }
       }
     }
   }
 
   /**
-   * Helper method to create AggregatedMetrics.
+   * Helper method to create PerformanceMetrics.
    *
    * @param registry the metrics registry
    * @param name metric name
@@ -68,12 +68,12 @@ public static <T> void initialize(T source, MetricsRegistry registry,
    * @param sampleName sample name
    * @param valueName value name
    * @param intervals intervals for quantiles
-   * @return an instance of AggregatedMetrics
+   * @return an instance of PerformanceMetrics
    */
-  private static AggregatedMetrics getAggregatedMetrics(
+  private static PerformanceMetrics getMetrics(
       MetricsRegistry registry, String name, String description,
       String sampleName, String valueName, int[] intervals) {
-    return new AggregatedMetrics(
+    return new PerformanceMetrics(
         registry.newStat(
             name, description, sampleName, valueName, false),
         MetricUtil.createQuantiles(
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index 76c766c82376..dfdcc67377a9 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
-import org.apache.hadoop.util.AggregatedMetrics;
+import org.apache.hadoop.util.PerformanceMetrics;
 import org.apache.hadoop.util.Time;
 
 /**
@@ -97,150 +97,150 @@ public final class S3GatewayMetrics implements MetricsSource {
 
   @Metric(about = "Latency for successfully retrieving an S3 bucket in " +
       "nanoseconds")
-  private AggregatedMetrics getBucketSuccessLatencyNs;
+  private PerformanceMetrics getBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve an S3 bucket in nanoseconds")
-  private AggregatedMetrics getBucketFailureLatencyNs;
+  private PerformanceMetrics getBucketFailureLatencyNs;
 
   @Metric(about = "Latency for successfully creating an S3 bucket in " +
       "nanoseconds")
-  private AggregatedMetrics createBucketSuccessLatencyNs;
+  private PerformanceMetrics createBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to create an S3 bucket in nanoseconds")
-  private AggregatedMetrics createBucketFailureLatencyNs;
+  private PerformanceMetrics createBucketFailureLatencyNs;
 
   @Metric(about = "Latency for successfully checking the existence of an " +
       "S3 bucket in nanoseconds")
-  private AggregatedMetrics headBucketSuccessLatencyNs;
+  private PerformanceMetrics headBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for successfully deleting an S3 bucket in " +
       "nanoseconds")
-  private AggregatedMetrics deleteBucketSuccessLatencyNs;
+  private PerformanceMetrics deleteBucketSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to delete an S3 bucket in nanoseconds")
-  private AggregatedMetrics deleteBucketFailureLatencyNs;
+  private PerformanceMetrics deleteBucketFailureLatencyNs;
 
   @Metric(about = "Latency for successfully retrieving an S3 bucket ACL " +
       "in nanoseconds")
-  private AggregatedMetrics getAclSuccessLatencyNs;
+  private PerformanceMetrics getAclSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve an S3 bucket ACL " +
       "in nanoseconds")
-  private AggregatedMetrics getAclFailureLatencyNs;
+  private PerformanceMetrics getAclFailureLatencyNs;
 
   @Metric(about = "Latency for successfully setting an S3 bucket ACL " +
       "in nanoseconds")
-  private AggregatedMetrics putAclSuccessLatencyNs;
+  private PerformanceMetrics putAclSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to set an S3 bucket ACL " +
       "in nanoseconds")
-  private AggregatedMetrics putAclFailureLatencyNs;
+  private PerformanceMetrics putAclFailureLatencyNs;
 
   @Metric(about = "Latency for successfully listing multipart uploads " +
       "in nanoseconds")
-  private AggregatedMetrics listMultipartUploadsSuccessLatencyNs;
+  private PerformanceMetrics listMultipartUploadsSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to list multipart uploads " +
       "in nanoseconds")
-  private AggregatedMetrics listMultipartUploadsFailureLatencyNs;
+  private PerformanceMetrics listMultipartUploadsFailureLatencyNs;
 
   // RootEndpoint
 
   @Metric(about = "Latency for successfully listing S3 buckets " +
       "in nanoseconds")
-  private AggregatedMetrics listS3BucketsSuccessLatencyNs;
+  private PerformanceMetrics listS3BucketsSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to list S3 buckets " +
       "in nanoseconds")
-  private AggregatedMetrics listS3BucketsFailureLatencyNs;
+  private PerformanceMetrics listS3BucketsFailureLatencyNs;
 
   // ObjectEndpoint
 
   @Metric(about = "Latency for successfully creating a multipart object key " +
       "in nanoseconds")
-  private AggregatedMetrics createMultipartKeySuccessLatencyNs;
+  private PerformanceMetrics createMultipartKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to create a multipart object key in " +
       "nanoseconds")
-  private AggregatedMetrics createMultipartKeyFailureLatencyNs;
+  private PerformanceMetrics createMultipartKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully copying an S3 object in " +
       "nanoseconds")
-  private AggregatedMetrics copyObjectSuccessLatencyNs;
+  private PerformanceMetrics copyObjectSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to copy an S3 object in nanoseconds")
-  private AggregatedMetrics copyObjectFailureLatencyNs;
+  private PerformanceMetrics copyObjectFailureLatencyNs;
 
   @Metric(about = "Latency for successfully creating an S3 object key in " +
       "nanoseconds")
-  private AggregatedMetrics createKeySuccessLatencyNs;
+  private PerformanceMetrics createKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to create an S3 object key in " +
       "nanoseconds")
-  private AggregatedMetrics createKeyFailureLatencyNs;
+  private PerformanceMetrics createKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully listing parts of a multipart " +
       "upload in nanoseconds")
-  private AggregatedMetrics listPartsSuccessLatencyNs;
+  private PerformanceMetrics listPartsSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to list parts of a multipart upload " +
       "in nanoseconds")
-  private AggregatedMetrics listPartsFailureLatencyNs;
+  private PerformanceMetrics listPartsFailureLatencyNs;
 
   @Metric(about = "Latency for successfully retrieving an S3 object in " +
       "nanoseconds")
-  private AggregatedMetrics getKeySuccessLatencyNs;
+  private PerformanceMetrics getKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve an S3 object in nanoseconds")
-  private AggregatedMetrics getKeyFailureLatencyNs;
+  private PerformanceMetrics getKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully retrieving metadata for an S3 " +
       "object in nanoseconds")
-  private AggregatedMetrics headKeySuccessLatencyNs;
+  private PerformanceMetrics headKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to retrieve metadata for an S3 object " +
       "in nanoseconds")
-  private AggregatedMetrics headKeyFailureLatencyNs;
+  private PerformanceMetrics headKeyFailureLatencyNs;
 
   @Metric(about = "Latency for successfully initiating a multipart upload in " +
       "nanoseconds")
-  private AggregatedMetrics initMultipartUploadSuccessLatencyNs;
+  private PerformanceMetrics initMultipartUploadSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to initiate a multipart upload in " +
       "nanoseconds")
-  private AggregatedMetrics initMultipartUploadFailureLatencyNs;
+  private PerformanceMetrics initMultipartUploadFailureLatencyNs;
 
   @Metric(about = "Latency for successfully completing a multipart upload in " +
       "nanoseconds")
-  private AggregatedMetrics completeMultipartUploadSuccessLatencyNs;
+  private PerformanceMetrics completeMultipartUploadSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to complete a multipart upload in " +
       "nanoseconds")
-  private AggregatedMetrics completeMultipartUploadFailureLatencyNs;
+  private PerformanceMetrics completeMultipartUploadFailureLatencyNs;
 
   @Metric(about = "Latency for successfully aborting a multipart upload in " +
       "nanoseconds")
-  private AggregatedMetrics abortMultipartUploadSuccessLatencyNs;
+  private PerformanceMetrics abortMultipartUploadSuccessLatencyNs;
 
   @Metric(about = "Latency for failing to abort a multipart upload in " +
       "nanoseconds")
-  private AggregatedMetrics abortMultipartUploadFailureLatencyNs;
+  private PerformanceMetrics abortMultipartUploadFailureLatencyNs;
 
   @Metric(about = "Latency for successfully deleting an S3 object in " +
       "nanoseconds")
-  private AggregatedMetrics deleteKeySuccessLatencyNs;
+  private PerformanceMetrics deleteKeySuccessLatencyNs;
 
   @Metric(about = "Latency for failing to delete an S3 object in nanoseconds")
-  private AggregatedMetrics deleteKeyFailureLatencyNs;
+  private PerformanceMetrics deleteKeyFailureLatencyNs;
 
   @Metric(about = "Latency for put metadata of an key in nanoseconds")
-  private AggregatedMetrics putKeyMetadataLatencyNs;
+  private PerformanceMetrics putKeyMetadataLatencyNs;
 
   @Metric(about = "Latency for get metadata of an key in nanoseconds")
-  private AggregatedMetrics getKeyMetadataLatencyNs;
+  private PerformanceMetrics getKeyMetadataLatencyNs;
 
   @Metric(about = "Latency for copy metadata of an key in nanoseconds")
-  private AggregatedMetrics copyKeyMetadataLatencyNs;
+  private PerformanceMetrics copyKeyMetadataLatencyNs;
 
   /**
    * Private constructor.
@@ -249,7 +249,7 @@ private S3GatewayMetrics(OzoneConfiguration conf) {
     this.registry = new MetricsRegistry(SOURCE_NAME);
     int[] intervals = conf.getInts(S3GatewayConfigKeys
         .OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
-    AggregatedMetrics.initializeMetrics(
+    PerformanceMetrics.initializeMetrics(
         this, registry, "Ops", "Time", intervals);
   }
 

From 75da05fb1cde8bd22ec0fd01b1963d3741c9470a Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Mon, 20 Nov 2023 12:06:00 +0800
Subject: [PATCH 04/12] add OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY
 to addPropertiesNotInXml

---
 .../org/apache/hadoop/ozone/TestOzoneConfigurationFields.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index cb29d61e1a4c..2ebfdac584b0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -141,6 +141,7 @@ private void addPropertiesNotInXml() {
         ScmConfigKeys.OZONE_SCM_PIPELINE_PLACEMENT_IMPL_KEY,
         ScmConfigKeys.OZONE_SCM_HA_PREFIX,
         S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED,
+        S3GatewayConfigKeys.OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY,
         HddsConfigKeys.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT
     ));
   }

From 1589e17d0ab64d12064050c73a75bfba52b62132 Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Mon, 20 Nov 2023 13:15:11 +0800
Subject: [PATCH 05/12] Add documentation for initializeMetrics() intervals

---
 .../java/org/apache/hadoop/util/PerformanceMetrics.java     | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java
index 4cc645f2a53c..a470ff9657f7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/PerformanceMetrics.java
@@ -40,7 +40,9 @@ public class PerformanceMetrics {
    *
    * @param source the metrics source
    * @param registry the metrics registry
-   * @param intervals the intervals for quantiles computation
+   * @param intervals the intervals for quantiles computation. Note, each
+   *        interval in 'intervals' increases memory usage, as it corresponds
+   *        to a separate quantile calculator.
    */
   public static synchronized <T> void initializeMetrics(T source,
       MetricsRegistry registry, String sampleName, String valueName,
@@ -54,7 +56,7 @@ public static synchronized <T> void initializeMetrics(T source,
   }
 
   /**
-   * Constructs an instance of PerformanceMetrics with the specified MutableStat,
+   * Construct an instance of PerformanceMetrics with the specified MutableStat,
    * MutableQuantiles, and MutableMinMax.
    *
    * @param stat the stat metric

From f0df6158f7479554797f3bfcd726a9db202f8468 Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Tue, 21 Nov 2023 00:35:17 +0800
Subject: [PATCH 06/12] findbugs

---
 hadoop-hdds/common/dev-support/findbugsExcludeFile.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml b/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
index ec4cd0ac5a7f..b2e94a5c6f67 100644
--- a/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
@@ -31,6 +31,10 @@
     <Method name="update" />
     <Bug pattern="SF_SWITCH_FALLTHROUGH,SF_SWITCH_NO_DEFAULT" />
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.hdds.utils.db.CodecBufferr" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
 
   <!-- Test -->
   <Match>

From 1afb2694be19cd298f8d41ea32b11b10cb2ee1bd Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Tue, 21 Nov 2023 17:42:51 +0800
Subject: [PATCH 07/12] revert findbugs excluding

---
 hadoop-hdds/common/dev-support/findbugsExcludeFile.xml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml b/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
index b2e94a5c6f67..ec4cd0ac5a7f 100644
--- a/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdds/common/dev-support/findbugsExcludeFile.xml
@@ -31,10 +31,6 @@
     <Method name="update" />
     <Bug pattern="SF_SWITCH_FALLTHROUGH,SF_SWITCH_NO_DEFAULT" />
   </Match>
-  <Match>
-    <Class name="org.apache.hadoop.hdds.utils.db.CodecBufferr" />
-    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
-  </Match>
 
   <!-- Test -->
   <Match>

From fe9191e161ab53a593e9eb49d39f80c81d7baad0 Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Tue, 21 Nov 2023 19:00:59 +0800
Subject: [PATCH 08/12] Fix thread safety issues for MutableMinMax

---
 .../src/main/java/org/apache/hadoop/util/MutableMinMax.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java
index b7d0d284ec3c..1f056ed742b3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java
@@ -79,7 +79,7 @@ private MinMax lastMinMax() {
   }
 
   @Override
-  public void snapshot(MetricsRecordBuilder builder, boolean all) {
+  public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || this.changed()) {
       builder.addGauge(iMinInfo, lastMinMax().min());
       builder.addGauge(iMaxInfo, lastMinMax().max());

From 56f944ec3ef4241fdc2e900a188727456d9baab9 Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Tue, 21 Nov 2023 23:17:08 +0800
Subject: [PATCH 09/12] Fix unit test

---
 .../apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
index 51b44998f00a..33f016499cbd 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -82,6 +83,7 @@ public void setup() {
     Mockito.when(client.getConfiguration()).thenReturn(conf);
     headers = Mockito.mock(HttpHeaders.class);
     clientProtocol = Mockito.mock(ClientProtocol.class);
+    S3GatewayMetrics.create(conf);
     Mockito.when(client.getProxy()).thenReturn(clientProtocol);
   }
 

From 0c4fe0ccf5279e94eea43a6ae4df12f4b778d75a Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Wed, 22 Nov 2023 00:44:08 +0800
Subject: [PATCH 10/12] Fix test

---
 .../java/org/apache/hadoop/ozone/TestMultipartObjectGet.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMultipartObjectGet.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMultipartObjectGet.java
index 37cc7a3411f0..01f9df42021a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMultipartObjectGet.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMultipartObjectGet.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.ozone.s3.endpoint.MultipartUploadInitiateResponse;
 import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
 import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -101,6 +102,7 @@ public static void init() throws Exception {
     REST.setClient(client);
     REST.setOzoneConfiguration(conf);
     REST.setContext(context);
+    S3GatewayMetrics.create(conf);
   }
 
   private static void startCluster()

From 5f66dab9d4a5b3c572a0632e32a576be73da413c Mon Sep 17 00:00:00 2001
From: xichen01 <mrchenx@126.com>
Date: Sun, 3 Dec 2023 03:07:21 +0800
Subject: [PATCH 11/12] Fix compile issue

---
 .../ozone/s3/endpoint/ObjectEndpointStreaming.java   | 12 +++++-------
 .../hadoop/ozone/s3/metrics/S3GatewayMetrics.java    |  2 +-
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index 3924dc10ae00..e509acb05bdb 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -52,6 +52,7 @@ final class ObjectEndpointStreaming {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(ObjectEndpointStreaming.class);
+  private static final S3GatewayMetrics METRICS = S3GatewayMetrics.getMetrics();
 
   private ObjectEndpointStreaming() {
   }
@@ -99,13 +100,12 @@ public static Pair<String, Long> putKeyWithStream(
       Map<String, String> keyMetadata,
       DigestInputStream body, PerformanceStringBuilder perf)
       throws IOException {
-    S3GatewayMetrics metrics = S3GatewayMetrics.create();
     long startNanos = Time.monotonicNowNanos();
     long writeLen;
     String eTag;
     try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath,
         length, replicationConfig, keyMetadata)) {
-      long metadataLatencyNs = metrics.updatePutKeyMetadataStats(startNanos);
+      long metadataLatencyNs = METRICS.updatePutKeyMetadataStats(startNanos);
       writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length);
       eTag = DatatypeConverter.printHexBinary(body.getMessageDigest().digest())
           .toLowerCase();
@@ -126,11 +126,10 @@ public static long copyKeyWithStream(
       InputStream body, PerformanceStringBuilder perf, long startNanos)
       throws IOException {
     long writeLen = 0;
-    S3GatewayMetrics metrics = S3GatewayMetrics.create();
     try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath,
         length, replicationConfig, keyMetadata)) {
       long metadataLatencyNs =
-          metrics.updateCopyKeyMetadataStats(startNanos);
+          METRICS.updateCopyKeyMetadataStats(startNanos);
       perf.appendMetaLatencyNanos(metadataLatencyNs);
       writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length);
     }
@@ -162,7 +161,6 @@ public static Response createMultipartKey(OzoneBucket ozoneBucket, String key,
       throws IOException, OS3Exception {
     long startNanos = Time.monotonicNowNanos();
     String eTag;
-    S3GatewayMetrics metrics = S3GatewayMetrics.getMetrics();
     // OmMultipartCommitUploadPartInfo can only be gotten after the
     // OzoneDataStreamOutput is closed, so we need to save the
     // KeyDataStreamOutput in the OzoneDataStreamOutput and use it to get the
@@ -171,13 +169,13 @@ public static Response createMultipartKey(OzoneBucket ozoneBucket, String key,
     try {
       try (OzoneDataStreamOutput streamOutput = ozoneBucket
           .createMultipartStreamKey(key, length, partNumber, uploadID)) {
-        long metadataLatencyNs = metrics.updatePutKeyMetadataStats(startNanos);
+        long metadataLatencyNs = METRICS.updatePutKeyMetadataStats(startNanos);
         long putLength =
             writeToStreamOutput(streamOutput, body, chunkSize, length);
         eTag = DatatypeConverter.printHexBinary(
             body.getMessageDigest().digest()).toLowerCase();
         ((KeyMetadataAware)streamOutput).getMetadata().put("ETag", eTag);
-        metrics.incPutKeySuccessLength(putLength);
+        METRICS.incPutKeySuccessLength(putLength);
         perf.appendMetaLatencyNanos(metadataLatencyNs);
         perf.appendSizeBytes(putLength);
         keyDataStreamOutput = streamOutput.getKeyDataStreamOutput();
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index 8f56090e80e7..ef7df24fecf8 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -725,7 +725,7 @@ public long getListS3BucketsFailure() {
     return listS3BucketsFailure.value();
   }
 
-  private long updateAndGetStats(MutableRate metric, long startNanos) {
+  private long updateAndGetStats(PerformanceMetrics metric, long startNanos) {
     long value = Time.monotonicNowNanos() - startNanos;
     metric.add(value);
     return value;

From be15e46cdddcb2e13f7252c3b001273bd4d1e609 Mon Sep 17 00:00:00 2001
From: "pony.chen" <pony.chen@shopee.com>
Date: Tue, 9 Jan 2024 02:31:45 +0800
Subject: [PATCH 12/12] Add a default value for configuration

---
 hadoop-hdds/common/src/main/resources/ozone-default.xml   | 8 ++++++++
 .../apache/hadoop/ozone/TestOzoneConfigurationFields.java | 5 ++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 079362f91666..f7a1e7ef66a1 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1876,6 +1876,14 @@
       will be used for http authentication.
     </description>
   </property>
+  <property>
+    <name>ozone.s3g.metrics.percentiles.intervals.seconds</name>
+    <value>60</value>
+    <tag>S3GATEWAY, PERFORMANCE</tag>
+    <description>Specifies the interval in seconds for the rollover of MutableQuantiles metrics.
+      Setting this interval equal to the metrics sampling time ensures more detailed metrics.
+    </description>
+  </property>
 
   <property>
     <name>ozone.om.save.metrics.interval</name>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 1865e3ebc642..1a437be8131b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -144,8 +144,7 @@ private void addPropertiesNotInXml() {
         HddsConfigKeys.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT,
         OzoneConfigKeys.HDDS_SCM_CLIENT_RPC_TIME_OUT,
         OzoneConfigKeys.HDDS_SCM_CLIENT_MAX_RETRY_TIMEOUT,
-        OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY,
-        S3GatewayConfigKeys.OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY
-        ));
+        OzoneConfigKeys.HDDS_SCM_CLIENT_FAILOVER_MAX_RETRY
+    ));
   }
 }