diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index 26d82443d6a6..78d10acd6d75 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -21,6 +21,7 @@ import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableGaugeFloat; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableRate; /** @@ -151,6 +152,15 @@ public static void unregister() { @Metric(about = "ACLs check in getObjectTagging") private MutableRate getObjectTaggingAclCheckLatencyNs; + @Metric(about = "Latency of each iteration of DirectoryDeletingService in ms") + private MutableGaugeLong directoryDeletingServiceLatencyMs; + + @Metric(about = "Latency of each iteration of KeyDeletingService in ms") + private MutableGaugeLong keyDeletingServiceLatencyMs; + + @Metric(about = "Latency of each iteration of OpenKeyCleanupService in ms") + private MutableGaugeLong openKeyCleanupServiceLatencyMs; + public void addLookupLatency(long latencyInNs) { lookupLatencyNs.add(latencyInNs); } @@ -299,4 +309,16 @@ public MutableRate getGetObjectTaggingAclCheckLatencyNs() { public void addGetObjectTaggingLatencyNs(long latencyInNs) { getObjectTaggingAclCheckLatencyNs.add(latencyInNs); } + + public void setDirectoryDeletingServiceLatencyMs(long latencyInMs) { + directoryDeletingServiceLatencyMs.set(latencyInMs); + } + + public void setKeyDeletingServiceLatencyMs(long latencyInMs) { + keyDeletingServiceLatencyMs.set(latencyInMs); + } + + public void setOpenKeyCleanupServiceLatencyMs(long latencyInMs) { + openKeyCleanupServiceLatencyMs.set(latencyInMs); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 66a1da44acc7..071f50367638 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMPerformanceMetrics; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -73,6 +74,7 @@ public abstract class AbstractKeyDeletingService extends BackgroundService private final OzoneManager ozoneManager; private final DeletingServiceMetrics metrics; + private final OMPerformanceMetrics perfMetrics; private final ScmBlockLocationProtocol scmClient; private final ClientId clientId = ClientId.randomId(); private final AtomicLong deletedDirsCount; @@ -94,6 +96,7 @@ public AbstractKeyDeletingService(String serviceName, long interval, this.movedFilesCount = new AtomicLong(0); this.runCount = new AtomicLong(0); this.metrics = ozoneManager.getDeletionMetrics(); + this.perfMetrics = ozoneManager.getPerfMetrics(); } protected int processKeyDeletes(List keyBlocksList, @@ -119,14 +122,15 @@ protected int processKeyDeletes(List keyBlocksList, LOG.info("{} BlockGroup deletion are acked by SCM in {} ms", keyBlocksList.size(), Time.monotonicNow() - startTime); if (blockDeletionResults != null) { - startTime = Time.monotonicNow(); + long purgeStartTime = Time.monotonicNow(); delCount = submitPurgeKeysRequest(blockDeletionResults, keysToModify, snapTableKey, expectedPreviousSnapshotId); int limit = ozoneManager.getConfiguration().getInt(OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK, OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); LOG.info("Blocks for {} (out of {}) keys are deleted from DB in {} ms. Limit per task is {}.", - delCount, blockDeletionResults.size(), Time.monotonicNow() - startTime, limit); + delCount, blockDeletionResults.size(), Time.monotonicNow() - purgeStartTime, limit); } + perfMetrics.setKeyDeletingServiceLatencyMs(Time.monotonicNow() - startTime); return delCount; } @@ -416,6 +420,7 @@ public void optimizeDirDeletesAndSubmitRequest( dirNum, subdirDelNum, subFileNum, (subDirNum - subdirDelNum), timeTakenInIteration, rnCnt); metrics.incrementDirectoryDeletionTotalMetrics(dirNum + subdirDelNum, subDirNum, subFileNum); + perfMetrics.setDirectoryDeletingServiceLatencyMs(timeTakenInIteration); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java index 60873f0fdea6..e10ca3f9a5a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java @@ -237,9 +237,11 @@ public BackgroundTaskResult call() throws Exception { }); } + long timeTaken = Time.monotonicNow() - startTime; LOG.info("Number of expired open keys submitted for deletion: {}," + " for commit: {}, cleanupLimit: {}, elapsed time: {}ms", - numOpenKeys, numHsyncKeys, cleanupLimitPerTask, Time.monotonicNow() - startTime); + numOpenKeys, numHsyncKeys, cleanupLimitPerTask, timeTaken); + ozoneManager.getPerfMetrics().setOpenKeyCleanupServiceLatencyMs(timeTaken); final int numKeys = numOpenKeys + numHsyncKeys; submittedOpenKeyCount.addAndGet(numKeys); return () -> numKeys;