diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java index f2f11025158d..a14adc80ddec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java @@ -19,6 +19,7 @@ import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableGaugeFloat; import org.apache.hadoop.metrics2.lib.MutableRate; /** @@ -113,12 +114,21 @@ public static void unregister() { @Metric(about = "Ratis local command execution latency in nano seconds") private MutableRate validateAndUpdateCacheLatencyNs; + @Metric(about = "average pagination for listKeys") + private MutableRate listKeysAveragePagination; + + @Metric(about = "ops per second for listKeys") + private MutableGaugeFloat listKeysOpsPerSec; + @Metric(about = "ACLs check latency in listKeys") private MutableRate listKeysAclCheckLatencyNs; @Metric(about = "resolveBucketLink latency in listKeys") private MutableRate listKeysResolveBucketLatencyNs; + @Metric(about = "readFromRockDb latency in listKeys") + private MutableRate listKeysReadFromRocksDbLatencyNs; + public void addLookupLatency(long latencyInNs) { lookupLatencyNs.add(latencyInNs); } @@ -216,6 +226,14 @@ public MutableRate getValidateAndUpdateCacheLatencyNs() { return validateAndUpdateCacheLatencyNs; } + public void setListKeysAveragePagination(long keyCount) { + listKeysAveragePagination.add(keyCount); + } + + public void setListKeysOpsPerSec(float opsPerSec) { + listKeysOpsPerSec.set(opsPerSec); + } + MutableRate getListKeysAclCheckLatencyNs() { return listKeysAclCheckLatencyNs; } @@ -223,4 +241,8 @@ MutableRate getListKeysAclCheckLatencyNs() { MutableRate getListKeysResolveBucketLatencyNs() { return listKeysResolveBucketLatencyNs; } + + public void addListKeysReadFromRocksDbLatencyNs(long latencyInNs) { + listKeysReadFromRocksDbLatencyNs.add(latencyInNs); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 6e25dc1f7f03..9f90643a17cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -317,6 +317,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private final Map tableCacheMetricsMap = new HashMap<>(); private SnapshotChainManager snapshotChainManager; + private final OMPerformanceMetrics perfMetrics; private final S3Batcher s3Batcher = new S3SecretBatcher(); /** @@ -328,7 +329,15 @@ public class OmMetadataManagerImpl implements OMMetadataManager, */ public OmMetadataManagerImpl(OzoneConfiguration conf, OzoneManager ozoneManager) throws IOException { + this(conf, ozoneManager, null); + } + + public OmMetadataManagerImpl(OzoneConfiguration conf, + OzoneManager ozoneManager, + OMPerformanceMetrics perfMetrics) + throws IOException { this.ozoneManager = ozoneManager; + this.perfMetrics = perfMetrics; this.lock = new OzoneManagerLock(conf); // TODO: This is a temporary check. Once fully implemented, all OM state // change should go through Ratis - be it standalone (for non-HA) or @@ -350,6 +359,7 @@ protected OmMetadataManagerImpl() { OzoneConfiguration conf = new OzoneConfiguration(); this.lock = new OzoneManagerLock(conf); this.omEpoch = 0; + perfMetrics = null; } public static OmMetadataManagerImpl createCheckpointMetadataManager( @@ -384,6 +394,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) setStore(loadDB(conf, dir, name, true, java.util.Optional.of(Boolean.TRUE), Optional.empty())); initializeOmTables(CacheType.PARTIAL_CACHE, false); + perfMetrics = null; } @@ -421,6 +432,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) stop(); throw e; } + perfMetrics = null; } @Override @@ -1163,7 +1175,7 @@ public List listBuckets(final String volumeName, public ListKeysResult listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException { - + long startNanos = Time.monotonicNowNanos(); List result = new ArrayList<>(); if (maxKeys <= 0) { return new ListKeysResult(result, false); @@ -1232,11 +1244,11 @@ public ListKeysResult listKeys(String volumeName, String bucketName, cacheKeyMap.put(key, omKeyInfo); } } - + long readFromRDbStartNs, readFromRDbStopNs = 0; // Get maxKeys from DB if it has. - try (TableIterator> keyIter = getKeyTable(getBucketLayout()).iterator()) { + readFromRDbStartNs = Time.monotonicNowNanos(); KeyValue< String, OmKeyInfo > kv; keyIter.seek(seekKey); // we need to iterate maxKeys + 1 here because if skipStartKey is true, @@ -1259,10 +1271,24 @@ public ListKeysResult listKeys(String volumeName, String bucketName, break; } } + readFromRDbStopNs = Time.monotonicNowNanos(); } boolean isTruncated = cacheKeyMap.size() > maxKeys; + if (perfMetrics != null) { + long keyCount; + if (isTruncated) { + keyCount = maxKeys; + } else { + keyCount = cacheKeyMap.size(); + } + perfMetrics.setListKeysAveragePagination(keyCount); + float opsPerSec = + keyCount / ((Time.monotonicNowNanos() - startNanos) / 1000000000.0f); + perfMetrics.setListKeysOpsPerSec(opsPerSec); + perfMetrics.addListKeysReadFromRocksDbLatencyNs(readFromRDbStopNs - readFromRDbStartNs); + } // Finally DB entries and cache entries are merged, then return the count // of maxKeys from the sorted map. currentCount = 0;