HDDS-8147. Introduce latency metrics for S3 Gateway operations

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/GenericCheckedSupplier.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+/**
+ * Similar to {@link java.util.function.Supplier}, this class presents a block
+ * of code generating a value with a possibility of throwing a generic
+ * Exception.
+ */
+
+@FunctionalInterface
+public interface GenericCheckedSupplier<T, E extends Exception> {
+  T get() throws E;
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java

@@ -39,6 +39,17 @@ public static <T, E extends IOException> T captureLatencyNs(
     }
   }
 
+  public static <T, E extends Exception> T captureLatencyNs(
+      MutableRate metric,
+      GenericCheckedSupplier<T, E> block) throws E {
+    long start = Time.monotonicNowNanos();
+    try {
+      return block.get();
+    } finally {
+      metric.add(Time.monotonicNowNanos() - start);
+    }
+  }
+
   public static <E extends IOException> void captureLatencyNs(
       MutableRate metric,
       CheckedRunnable<E> block) throws IOException {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -91,6 +91,7 @@
 import org.apache.hadoop.ozone.security.acl.RequestContext;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.CheckedSupplier;
 import org.apache.hadoop.util.Time;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -329,7 +330,7 @@ public OmKeyInfo lookupKey(OmKeyArgs args, String clientAddress)
     Preconditions.checkNotNull(args);
 
     OmKeyInfo value = captureLatencyNs(metrics.getLookupReadKeyInfoLatencyNs(),
-        () -> readKeyInfo(args));
+        (CheckedSupplier<OmKeyInfo, IOException>) () -> readKeyInfo(args));
 
     // If operation is head, do not perform any additional steps based on flags.
     // As head operation does not need any of those details.
@@ -1892,7 +1893,7 @@ public OmKeyInfo getKeyInfo(OmKeyArgs args, String clientAddress)
 
     OmKeyInfo value = captureLatencyNs(
         metrics.getGetKeyInfoReadKeyInfoLatencyNs(),
-        () -> readKeyInfo(args));
+        (CheckedSupplier<OmKeyInfo, IOException>) () -> readKeyInfo(args));
 
     // If operation is head, do not perform any additional steps based on flags.
     // As head operation does not need any of those details.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java

@@ -40,6 +40,7 @@
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
 import org.apache.hadoop.ozone.security.acl.RequestContext;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.CheckedSupplier;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
@@ -136,9 +137,10 @@ public OmMetadataReader(KeyManager keyManager,
   @Override
   public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException {
     long start = Time.monotonicNowNanos();
-    ResolvedBucket bucket = captureLatencyNs(
-        perfMetrics.getLookupResolveBucketLatencyNs(),
-        () -> ozoneManager.resolveBucketLink(args));
+    ResolvedBucket bucket =
+        captureLatencyNs(perfMetrics.getLookupResolveBucketLatencyNs(),
+            (CheckedSupplier<ResolvedBucket, IOException>) () -> ozoneManager
+                .resolveBucketLink(args));
     boolean auditSuccess = true;
     Map<String, String> auditMap = bucket.audit(args.toAuditMap());
 
@@ -190,9 +192,10 @@ public KeyInfoWithVolumeContext getKeyInfo(final OmKeyArgs args,
       resolvedVolumeArgs = args;
     }
 
-    final ResolvedBucket bucket = captureLatencyNs(
-        perfMetrics.getGetKeyInfoResolveBucketLatencyNs(),
-        () -> ozoneManager.resolveBucketLink(resolvedVolumeArgs));
+    final ResolvedBucket bucket =
+        captureLatencyNs(perfMetrics.getGetKeyInfoResolveBucketLatencyNs(),
+            (CheckedSupplier<ResolvedBucket, IOException>) () -> ozoneManager
+                .resolveBucketLink(resolvedVolumeArgs));
 
     boolean auditSuccess = true;
     OmKeyArgs resolvedArgs = bucket.update(resolvedVolumeArgs);

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java

@@ -26,6 +26,7 @@
 import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
 import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayLatencyMetrics;
 import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.apache.hadoop.ozone.util.OzoneNetUtils;
 import org.apache.hadoop.ozone.util.OzoneVersionInfo;
@@ -55,6 +56,7 @@ public class Gateway extends GenericCli {
 
   private S3GatewayHttpServer httpServer;
   private S3GatewayMetrics metrics;
+  private S3GatewayLatencyMetrics latencyMetrics;
   private OzoneConfiguration ozoneConfiguration;
 
   public static void main(String[] args) throws Exception {
@@ -72,6 +74,7 @@ public Void call() throws Exception {
     loginS3GUser(ozoneConfiguration);
     httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway");
     metrics = S3GatewayMetrics.create();
+    latencyMetrics = S3GatewayLatencyMetrics.create();
     start();
 
     ShutdownHookManager.get().addShutdownHook(() -> {
@@ -99,6 +102,7 @@ public void stop() throws Exception {
     LOG.info("Stopping Ozone S3 gateway");
     httpServer.stop();
     S3GatewayMetrics.unRegister();
+    S3GatewayLatencyMetrics.unRegister();
   }
 
   private static void loginS3GUser(OzoneConfiguration conf)

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -40,6 +40,8 @@
 import org.apache.hadoop.ozone.s3.util.ContinueToken;
 import org.apache.hadoop.ozone.s3.util.S3StorageType;
 import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
+import org.apache.hadoop.util.GenericCheckedSupplier;
+import org.apache.hadoop.util.Time;
 import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,6 +73,7 @@
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED;
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
+import static org.apache.hadoop.util.MetricUtil.captureLatencyNs;
 
 /**
  * Bucket level rest endpoints.
@@ -101,7 +104,7 @@ public Response get(
       @QueryParam("start-after") String startAfter,
       @QueryParam("uploads") String uploads,
       @QueryParam("acl") String aclMarker,
-      @Context HttpHeaders hh) throws OS3Exception, IOException {
+      @Context HttpHeaders hh) throws Exception {
     S3GAction s3GAction = S3GAction.GET_BUCKET;
     Iterator<? extends OzoneKey> ozoneKeyIterator;
     ContinueToken decodedToken =
@@ -110,7 +113,10 @@ public Response get(
     try {
       if (aclMarker != null) {
         s3GAction = S3GAction.GET_ACL;
-        S3BucketAcl result = getAcl(bucketName);
+        S3BucketAcl result =
+            captureLatencyNs(getLatencyMetrics().getGetAclLatencyNs(),
+                (GenericCheckedSupplier<S3BucketAcl, Exception>) () -> getAcl(
+                    bucketName));
         getMetrics().incGetAclSuccess();
         AUDIT.logReadSuccess(
             buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
@@ -130,8 +136,12 @@ public Response get(
       if (startAfter == null && marker != null) {
         startAfter = marker;
       }
-
-      OzoneBucket bucket = getBucket(bucketName);
+
+      OzoneBucket bucket =
+          captureLatencyNs(getLatencyMetrics().getGetBucketLatencyNs(),
+              (GenericCheckedSupplier<OzoneBucket, Exception>) () -> getBucket(
+                  bucketName));
+
       if (startAfter != null && continueToken != null) {
         // If continuation token and start after both are provided, then we
         // ignore start After
@@ -257,7 +267,7 @@ public Response get(
   public Response put(@PathParam("bucket") String bucketName,
                       @QueryParam("acl") String aclMarker,
                       @Context HttpHeaders httpHeaders,
-                      InputStream body) throws IOException, OS3Exception {
+                      InputStream body) throws Exception {
     S3GAction s3GAction = S3GAction.CREATE_BUCKET;
 
     try {
@@ -268,7 +278,9 @@ public Response put(@PathParam("bucket") String bucketName,
             buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
         return response;
       }
-      String location = createS3Bucket(bucketName);
+      String location =
+          captureLatencyNs(getLatencyMetrics().getCreateBucketLatencyNs(), (GenericCheckedSupplier<String, Exception>) () -> createS3Bucket(bucketName));
+
       AUDIT.logWriteSuccess(
           buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
       getMetrics().incCreateBucketSuccess();
@@ -292,6 +304,7 @@ public Response listMultipartUploads(
       @PathParam("bucket") String bucketName,
       @QueryParam("prefix") String prefix)
       throws OS3Exception, IOException {
+    long start = Time.monotonicNowNanos();
     S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD;
 
     OzoneBucket bucket = getBucket(bucketName);
@@ -313,6 +326,8 @@ public Response listMultipartUploads(
       AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction,
           getAuditParameters()));
       getMetrics().incListMultipartUploadsSuccess();
+      getLatencyMetrics().addListMultipartUploadsLatencyNs(
+          Time.monotonicNowNanos() - start);
       return Response.ok(result).build();
     } catch (OMException exception) {
       AUDIT.logReadFailure(
@@ -338,10 +353,12 @@ public Response listMultipartUploads(
    */
   @HEAD
   public Response head(@PathParam("bucket") String bucketName)
-      throws OS3Exception, IOException {
+      throws Exception {
     S3GAction s3GAction = S3GAction.HEAD_BUCKET;
     try {
-      getBucket(bucketName);
+      captureLatencyNs(getLatencyMetrics().getHeadBucketLatencyNs(),
+          (GenericCheckedSupplier<OzoneBucket, Exception>) () -> getBucket(
+              bucketName));
       AUDIT.logReadSuccess(
           buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
       getMetrics().incHeadBucketSuccess();
@@ -361,7 +378,7 @@ public Response head(@PathParam("bucket") String bucketName)
    */
   @DELETE
   public Response delete(@PathParam("bucket") String bucketName)
-      throws IOException, OS3Exception {
+      throws Exception {
     S3GAction s3GAction = S3GAction.DELETE_BUCKET;
 
     try {
@@ -500,6 +517,7 @@ public S3BucketAcl getAcl(String bucketName)
    */
   public Response putAcl(String bucketName, HttpHeaders httpHeaders,
                          InputStream body) throws IOException, OS3Exception {
+    long start = Time.monotonicNowNanos();
     String grantReads = httpHeaders.getHeaderString(S3Acl.GRANT_READ);
     String grantWrites = httpHeaders.getHeaderString(S3Acl.GRANT_WRITE);
     String grantReadACP = httpHeaders.getHeaderString(S3Acl.GRANT_READ_CAP);
@@ -579,6 +597,7 @@ public Response putAcl(String bucketName, HttpHeaders httpHeaders,
       for (OzoneAcl acl : ozoneAclListOnVolume) {
         volume.addAcl(acl);
       }
+      getLatencyMetrics().addPutAclLatencyNs(Time.monotonicNowNanos() - start);
     } catch (OMException exception) {
       getMetrics().incPutAclFailure();
       auditWriteFailure(S3GAction.PUT_ACL, exception);

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -56,6 +56,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayLatencyMetrics;
 import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.apache.hadoop.ozone.s3.util.AuditUtils;
 import org.slf4j.Logger;
@@ -360,6 +361,11 @@ public S3GatewayMetrics getMetrics() {
     return S3GatewayMetrics.create();
   }
 
+  @VisibleForTesting
+  public S3GatewayLatencyMetrics getLatencyMetrics() {
+    return S3GatewayLatencyMetrics.create();
+  }
+
   protected Map<String, String> getAuditParameters() {
     return AuditUtils.getAuditParameters(context);
   }