HDDS-9874. Introduce Metrics for listKeys Dashboard

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java

@@ -113,12 +113,21 @@ public static void unregister() {
   @Metric(about = "Ratis local command execution latency in nano seconds")
   private MutableRate validateAndUpdateCacheLatencyNs;
 
+  @Metric(about = "average pagination for listKeys")
+  private MutableRate listKeysAveragePagination;
+
+  @Metric(about = "ops per second for listKeys")
+  private MutableRate listKeysOpsPerSec;
+
   @Metric(about = "ACLs check latency in listKeys")
   private MutableRate listKeysAclCheckLatencyNs;
 
   @Metric(about = "resolveBucketLink latency in listKeys")
   private MutableRate listKeysResolveBucketLatencyNs;
 
+  @Metric(about = "readFromRockDb latency in listKeys")
+  private MutableRate listKeysReadFromRocksDbLatencyNs;
+
   public void addLookupLatency(long latencyInNs) {
     lookupLatencyNs.add(latencyInNs);
   }
@@ -216,11 +225,23 @@ public MutableRate getValidateAndUpdateCacheLatencyNs() {
     return validateAndUpdateCacheLatencyNs;
   }
 
+  public void setListKeysAveragePagination(long averagePagination) {
+    listKeysAveragePagination.add(averagePagination);
+  }
+
+  public void setListKeysOpsPerNs(long opsPerNs) {
+    listKeysOpsPerSec.add(opsPerNs);
+  }
+
   MutableRate getListKeysAclCheckLatencyNs() {
     return listKeysAclCheckLatencyNs;
   }
 
   MutableRate getListKeysResolveBucketLatencyNs() {
     return listKeysResolveBucketLatencyNs;
   }
+
+  public void addListKeysReadFromRocksDbLatencyNs(long latencyInNs) {
+    listKeysReadFromRocksDbLatencyNs.add(latencyInNs);
+  }
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -317,6 +317,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
   private final Map<String, TableCacheMetrics> tableCacheMetricsMap =
       new HashMap<>();
   private SnapshotChainManager snapshotChainManager;
+  private final OMPerformanceMetrics perfMetrics;
   private final S3Batcher s3Batcher = new S3SecretBatcher();
 
   /**
@@ -328,28 +329,46 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
    */
   public OmMetadataManagerImpl(OzoneConfiguration conf,
       OzoneManager ozoneManager) throws IOException {
-    this.ozoneManager = ozoneManager;
+    init(conf, ozoneManager);
+    this.lock = new OzoneManagerLock(conf);
+    this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled);
+    this.perfMetrics = null;
+    start(conf);
+  }
+
+  public OmMetadataManagerImpl(OzoneConfiguration conf,
+                               OzoneManager ozoneManager,
+                               OMPerformanceMetrics perfMetrics)
+      throws IOException {
+    init(conf, ozoneManager);
     this.lock = new OzoneManagerLock(conf);
-    // TODO: This is a temporary check. Once fully implemented, all OM state
-    //  change should go through Ratis - be it standalone (for non-HA) or
-    //  replicated (for HA).
-    isRatisEnabled = conf.getBoolean(
-        OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY,
-        OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT);
     this.omEpoch = OmUtils.getOMEpoch(isRatisEnabled);
+    this.perfMetrics = perfMetrics;
     // For test purpose only
     ignorePipelineinKey = conf.getBoolean(
-        "ozone.om.ignore.pipeline", Boolean.TRUE);
+  "ozone.om.ignore.pipeline", Boolean.TRUE);
     start(conf);
   }
-
+  private void init(OzoneConfiguration conf, OzoneManager manager) {
+    this.ozoneManager = manager;
+    // TODO: This is a temporary check. Once fully implemented, all OM state
+  //  change should go through Ratis - be it standalone (for non-HA) or
+  //  replicated (for HA).
+    isRatisEnabled = conf.getBoolean(
+      OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY,
+      OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT);
+  // For test purpose only
+    ignorePipelineinKey = conf.getBoolean(
+      "ozone.om.ignore.pipeline", Boolean.TRUE);
+  }
   /**
    * For subclass overriding.
    */
   protected OmMetadataManagerImpl() {
     OzoneConfiguration conf = new OzoneConfiguration();
     this.lock = new OzoneManagerLock(conf);
     this.omEpoch = 0;
+    perfMetrics = null;
   }
 
   public static OmMetadataManagerImpl createCheckpointMetadataManager(
@@ -384,6 +403,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
     setStore(loadDB(conf, dir, name, true,
         java.util.Optional.of(Boolean.TRUE), Optional.empty()));
     initializeOmTables(CacheType.PARTIAL_CACHE, false);
+    perfMetrics = null;
   }
 
 
@@ -421,6 +441,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
       stop();
       throw e;
     }
+    perfMetrics = null;
   }
 
   @Override
@@ -1163,7 +1184,7 @@ public List<OmBucketInfo> listBuckets(final String volumeName,
   public ListKeysResult listKeys(String volumeName, String bucketName,
                                  String startKey, String keyPrefix, int maxKeys)
       throws IOException {
-
+    long startNanos = Time.monotonicNowNanos();
     List<OmKeyInfo> result = new ArrayList<>();
     if (maxKeys <= 0) {
       return new ListKeysResult(result, false);
@@ -1232,11 +1253,11 @@ public ListKeysResult listKeys(String volumeName, String bucketName,
         cacheKeyMap.put(key, omKeyInfo);
       }
     }
-
+    long readFromRDbStartNs, readFromRDbStopNs = 0;
     // Get maxKeys from DB if it has.
-
     try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>>
              keyIter = getKeyTable(getBucketLayout()).iterator()) {
+      readFromRDbStartNs = Time.monotonicNowNanos();
       KeyValue< String, OmKeyInfo > kv;
       keyIter.seek(seekKey);
       // we need to iterate maxKeys + 1 here because if skipStartKey is true,
@@ -1259,10 +1280,24 @@ public ListKeysResult listKeys(String volumeName, String bucketName,
           break;
         }
       }
+      readFromRDbStopNs = Time.monotonicNowNanos();
     }
 
     boolean isTruncated = cacheKeyMap.size() > maxKeys;
 
+    if (perfMetrics != null) {
+      long averagePagination;
+      if (isTruncated) {
+        averagePagination = maxKeys;
+      } else {
+        averagePagination = cacheKeyMap.size();
+      }
+      perfMetrics.setListKeysAveragePagination(averagePagination);
+      long opsPerNs =
+          averagePagination / (Time.monotonicNowNanos() - startNanos);
+      perfMetrics.setListKeysOpsPerNs(opsPerNs);
+      perfMetrics.addListKeysReadFromRocksDbLatencyNs(readFromRDbStopNs - readFromRDbStartNs);
+    }
     // Finally DB entries and cache entries are merged, then return the count
     // of maxKeys from the sorted map.
     currentCount = 0;

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java

@@ -260,7 +260,8 @@ private void addBucketToMetaTable(OmBucketInfo bucketInfo)
    * @throws IOException if I/O error occurs in setting up data store for the
    *                     metadata manager.
    */
-  private OMMetadataManager createOMMetadataManagerSpy() throws IOException {
+  private OmMetadataManagerImpl createOMMetadataManagerSpy()
+      throws IOException {
     OzoneConfiguration conf = new OzoneConfiguration();
     File newFolder = folder.toFile();
     if (!newFolder.exists()) {

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/PrefixParser.java

@@ -128,9 +128,9 @@ public void parse(String vol, String buck, String db,
 
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, db);
-
     OmMetadataManagerImpl metadataManager =
         new OmMetadataManagerImpl(conf, null);
+
     metadataManager.start(conf);
 
     org.apache.hadoop.fs.Path effectivePath =

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java

@@ -79,6 +79,11 @@ public static void main(String[] args) throws IOException {
     }
     OmMetadataManagerImpl metadataManager =
         new OmMetadataManagerImpl(configuration, null);
+    return getBlockIdDetailsUtils(metadataManager);
+  }
+
+  private static Map<Long, List<Map<Long, BlockIdDetails>>> getBlockIdDetailsUtils(
+      OmMetadataManagerImpl metadataManager) throws IOException {
     try {
       Table<String, OmKeyInfo> keyTable =
           metadataManager.getKeyTable(getBucketLayout());