HDDS-13510. Add latency metrics for key creation operation

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java

@@ -148,6 +148,21 @@ public class OMPerformanceMetrics {
   @Metric(about = "Latency of each iteration of OpenKeyCleanupService in ms")
   private MutableGaugeLong openKeyCleanupServiceLatencyMs;
 
+  @Metric(about = "ResolveBucketLink and ACL check latency for createKey in nanoseconds")
+  private MutableRate createKeyResolveBucketAndAclCheckLatencyNs;
+
+  @Metric(about = "check quota for createKey in nanoseconds")
+  private MutableRate createKeyQuotaCheckLatencyNs;
+
+  @Metric(about = "Block allocation latency for createKey in nanoseconds")
+  private MutableRate createKeyAllocateBlockLatencyNs;
+
+  @Metric(about = "createKeyFailure latency in nanoseconds")
+  private MutableRate createKeyFailureLatencyNs;
+
+  @Metric(about = "creteKeySuccess latency in nanoseconds")
+  private MutableRate createKeySuccessLatencyNs;
+
   public static OMPerformanceMetrics register() {
     MetricsSystem ms = DefaultMetricsSystem.instance();
     return ms.register(SOURCE_NAME,
@@ -291,6 +306,26 @@ public void setDeleteKeysAclCheckLatencyNs(long latencyInNs) {
   public MutableRate getDeleteKeyResolveBucketAndAclCheckLatencyNs() {
     return deleteKeyResolveBucketAndAclCheckLatencyNs;
   }
+
+  public MutableRate getCreateKeyResolveBucketAndAclCheckLatencyNs() {
+    return createKeyResolveBucketAndAclCheckLatencyNs;
+  }
+
+  public void addCreateKeyQuotaCheckLatencyNs(long latencyInNs) {
+    createKeyQuotaCheckLatencyNs.add(latencyInNs);
+  }
+
+  public MutableRate getCreateKeyAllocateBlockLatencyNs() {
+    return createKeyAllocateBlockLatencyNs;
+  }
+
+  public void addCreateKeyFailureLatencyNs(long latencyInNs) {
+    createKeyFailureLatencyNs.add(latencyInNs);
+  }
+
+  public void addCreateKeySuccessLatencyNs(long latencyInNs) {
+    createKeySuccessLatencyNs.add(latencyInNs);
+  }
 
   public void addListKeysReadFromRocksDbLatencyNs(long latencyInNs) {
     listKeysReadFromRocksDbLatencyNs.add(latencyInNs);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java

@@ -20,6 +20,7 @@
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
 import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS;
 import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.FILE_EXISTS_IN_GIVENPATH;
+import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;
 
 import com.google.common.base.Preconditions;
 import java.io.IOException;
@@ -38,6 +39,7 @@
 import org.apache.hadoop.ozone.audit.OMAction;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
 import org.apache.hadoop.ozone.om.OzoneConfigUtil;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -89,6 +91,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
 
     KeyArgs keyArgs = createKeyRequest.getKeyArgs();
 
+    final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics();
+
     if (keyArgs.hasExpectedDataGeneration()) {
       ozoneManager.checkFeatureEnabled(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
     }
@@ -141,15 +145,16 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
       //  till leader is identified.
       UserInfo userInfo = getUserInfo();
       List<OmKeyLocationInfo> omKeyLocationInfoList =
-          allocateBlock(ozoneManager.getScmClient(),
-              ozoneManager.getBlockTokenSecretManager(), repConfig,
-              new ExcludeList(), requestedSize, scmBlockSize,
-              ozoneManager.getPreallocateBlocksMax(),
-              ozoneManager.isGrpcBlockTokenEnabled(),
-              ozoneManager.getOMServiceId(),
-              ozoneManager.getMetrics(),
-              keyArgs.getSortDatanodes(),
-              userInfo);
+          captureLatencyNs(perfMetrics.getCreateKeyAllocateBlockLatencyNs(),
+              () -> allocateBlock(ozoneManager.getScmClient(),
+                ozoneManager.getBlockTokenSecretManager(), repConfig,
+                new ExcludeList(), requestedSize, scmBlockSize,
+                ozoneManager.getPreallocateBlocksMax(),
+                ozoneManager.isGrpcBlockTokenEnabled(),
+                ozoneManager.getOMServiceId(),
+                ozoneManager.getMetrics(),
+                keyArgs.getSortDatanodes(),
+                userInfo));
 
       newKeyArgs = keyArgs.toBuilder().setModificationTime(Time.now())
               .setType(type).setFactor(factor)
@@ -171,9 +176,11 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
       generateRequiredEncryptionInfo(keyArgs, newKeyArgs, ozoneManager);
     }
 
+    KeyArgs.Builder finalNewKeyArgs = newKeyArgs;
     KeyArgs resolvedKeyArgs =
-        resolveBucketAndCheckKeyAcls(newKeyArgs.build(), ozoneManager,
-            IAccessAuthorizer.ACLType.CREATE);
+        captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), 
+            () -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager,
+            IAccessAuthorizer.ACLType.CREATE));
     newCreateKeyRequest =
         createKeyRequest.toBuilder().setKeyArgs(resolvedKeyArgs)
             .setClientID(UniqueId.next());
@@ -212,6 +219,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
     Result result = null;
     List<OmKeyInfo> missingParentInfos = null;
     int numMissingParents = 0;
+    final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics();
+    long createKeyStartTime = Time.monotonicNowNanos();
     try {
 
       mergeOmLockDetails(
@@ -302,9 +311,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
           * ozoneManager.getScmBlockSize()
           * replicationConfig.getRequiredNodes();
       // check bucket and volume quota
+      long quotaCheckStartTime = Time.monotonicNowNanos();
       checkBucketQuotaInBytes(omMetadataManager, bucketInfo,
           preAllocatedSpace);
       checkBucketQuotaInNamespace(bucketInfo, numMissingParents + 1L);
+      perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() - quotaCheckStartTime);
       bucketInfo.incrUsedNamespace(numMissingParents);
 
       if (numMissingParents > 0) {
@@ -340,6 +351,14 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
       omClientResponse = new OMKeyCreateResponse(
           createErrorOMResponse(omResponse, exception), getBucketLayout());
     } finally {
+      long createKeyLatency = Time.monotonicNowNanos() - createKeyStartTime;
+
+      if (Result.SUCCESS.equals(result)) {
+        perfMetrics.addCreateKeySuccessLatencyNs(createKeyLatency);
+      } else {
+        perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency);
+      }
+
       if (acquireLock) {
         mergeOmLockDetails(ozoneLockStrategy
             .releaseWriteLock(omMetadataManager, volumeName,

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.ozone.audit.OMAction;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
 import org.apache.hadoop.ozone.om.OzoneConfigUtil;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -50,6 +51,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateKeyResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.util.Time;
 
 /**
  * Handles CreateKey request layout version1.
@@ -88,9 +90,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
     OzoneManagerProtocolProtos.OMResponse.Builder omResponse =
             OmResponseUtil.getOMResponseBuilder(getOmRequest());
     Exception exception = null;
-    Result result;
+    Result result = null;
     List<OmDirectoryInfo> missingParentInfos;
     int numKeysCreated = 0;
+    final OMPerformanceMetrics perfMetrics = ozoneManager.getPerfMetrics();
+    long createKeyStartTime = Time.monotonicNowNanos();
     try {
       mergeOmLockDetails(omMetadataManager.getLock()
           .acquireWriteLock(BUCKET_LOCK, volumeName, bucketName));
@@ -174,9 +178,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
       long preAllocatedSpace =
           newLocationList.size() * ozoneManager.getScmBlockSize() * repConfig
               .getRequiredNodes();
+      long quotaCheckStartTime = Time.monotonicNowNanos();
       checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
           preAllocatedSpace);
       checkBucketQuotaInNamespace(omBucketInfo, numKeysCreated + 1L);
+      perfMetrics.addCreateKeyQuotaCheckLatencyNs(Time.monotonicNowNanos() - quotaCheckStartTime);
       omBucketInfo.incrUsedNamespace(numKeysCreated);
 
       // Add to cache entry can be done outside of lock for this openKey.
@@ -214,6 +220,14 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
       omClientResponse = new OMKeyCreateResponseWithFSO(
               createErrorOMResponse(omResponse, exception), getBucketLayout());
     } finally {
+      long createKeyLatency = Time.monotonicNowNanos() - createKeyStartTime;
+
+      if (Result.SUCCESS.equals(result)) {
+        perfMetrics.addCreateKeySuccessLatencyNs(createKeyLatency);
+      } else {
+        perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency);
+      }
+
       if (acquireLock) {
         mergeOmLockDetails(omMetadataManager.getLock()
             .releaseWriteLock(BUCKET_LOCK, volumeName, bucketName));

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java

@@ -53,6 +53,7 @@
 import org.apache.hadoop.ozone.audit.AuditLogger;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
 import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -99,6 +100,9 @@ public class TestCleanupTableInfo {
   @Mock
   private OMMetrics omMetrics;
 
+  @Mock
+  private OMPerformanceMetrics perfMetrics;
+
   @Mock
   private OzoneManager om;
 
@@ -184,6 +188,7 @@ public void testKeyCreateRequestSetsAllTouchedTableCachesForEviction() {
     when(om.getEnableFileSystemPaths()).thenReturn(true);
     when(om.getOzoneLockProvider()).thenReturn(
         new OzoneLockProvider(false, false));
+    when(om.getPerfMetrics()).thenReturn(perfMetrics);
 
     Map<String, Integer> cacheItemCount = recordCacheItemCounts();