HDDS-4363. Add metric to track the number of RocksDB open/close operations.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java

@@ -22,6 +22,7 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.Striped;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -33,6 +34,7 @@
 import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
 import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaOneImpl;
 import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl;
+import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,6 +48,7 @@ public final class ContainerCache extends LRUMap {
   private static ContainerCache cache;
   private static final float LOAD_FACTOR = 0.75f;
   private final Striped<Lock> rocksDBLock;
+  private static ContainerCacheMetrics metrics;
   /**
    * Constructs a cache that holds DBHandle references.
    */
@@ -55,6 +58,11 @@ private ContainerCache(int maxSize, int stripes, float loadFactor, boolean
     rocksDBLock = Striped.lazyWeakLock(stripes);
   }
 
+  @VisibleForTesting
+  public ContainerCacheMetrics getMetrics() {
+    return metrics;
+  }
+
   /**
    * Return a singleton instance of {@link ContainerCache}
    * that holds the DB handlers.
@@ -71,6 +79,7 @@ public synchronized static ContainerCache getInstance(
           OzoneConfigKeys.OZONE_CONTAINER_CACHE_LOCK_STRIPES,
           OzoneConfigKeys.OZONE_CONTAINER_CACHE_LOCK_STRIPES_DEFAULT);
       cache = new ContainerCache(cacheSize, stripes, LOAD_FACTOR, true);
+      metrics = ContainerCacheMetrics.create();
     }
     return cache;
   }
@@ -86,7 +95,7 @@ public void shutdownCache() {
       while (iterator.hasNext()) {
         iterator.next();
         ReferenceCountedDB db = (ReferenceCountedDB) iterator.getValue();
-        Preconditions.checkArgument(db.cleanup(), "refCount:",
+        Preconditions.checkArgument(cleanupDb(db), "refCount:",
             db.getReferenceCount());
       }
       // reset the cache
@@ -104,7 +113,8 @@ protected boolean removeLRU(LinkEntry entry) {
     ReferenceCountedDB db = (ReferenceCountedDB) entry.getValue();
     lock.lock();
     try {
-      return db.cleanup();
+      metrics.incNumCacheEvictions();
+      return cleanupDb(db);
     } finally {
       lock.unlock();
     }
@@ -130,19 +140,24 @@ public ReferenceCountedDB getDB(long containerID, String containerDBType,
     ReferenceCountedDB db;
     Lock containerLock = rocksDBLock.get(containerDBPath);
     containerLock.lock();
+    metrics.incNumDbGetOps();
     try {
       lock.lock();
       try {
         db = (ReferenceCountedDB) this.get(containerDBPath);
         if (db != null) {
+          metrics.incNumCacheHits();
           db.incrementReference();
           return db;
+        } else {
+          metrics.incNumCacheMisses();
         }
       } finally {
         lock.unlock();
       }
 
       try {
+        long start = Time.monotonicNow();
         DatanodeStore store;
 
         if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
@@ -157,6 +172,7 @@ public ReferenceCountedDB getDB(long containerID, String containerDBType,
         }
 
         db = new ReferenceCountedDB(store, containerDBPath);
+        metrics.incDbOpenLatency(Time.monotonicNow() - start);
       } catch (Exception e) {
         LOG.error("Error opening DB. Container:{} ContainerPath:{}",
             containerID, containerDBPath, e);
@@ -171,7 +187,7 @@ public ReferenceCountedDB getDB(long containerID, String containerDBType,
           // increment the reference before returning the object
           currentDB.incrementReference();
           // clean the db created in previous step
-          db.cleanup();
+          cleanupDb(db);
           return currentDB;
         } else {
           this.put(containerDBPath, db);
@@ -197,7 +213,7 @@ public void removeDB(String containerDBPath) {
     try {
       ReferenceCountedDB db = (ReferenceCountedDB)this.get(containerDBPath);
       if (db != null) {
-        Preconditions.checkArgument(db.cleanup(), "refCount:",
+        Preconditions.checkArgument(cleanupDb(db), "refCount:",
             db.getReferenceCount());
       }
       this.remove(containerDBPath);
@@ -206,6 +222,15 @@ public void removeDB(String containerDBPath) {
     }
   }
 
+  private boolean cleanupDb(ReferenceCountedDB db) {
+    long time = Time.monotonicNow();
+    boolean ret = db.cleanup();
+    if (ret) {
+      metrics.incDbCloseLatency(Time.monotonicNow() - time);
+    }
+    return ret;
+  }
+
   /**
    * Add a DB handler into cache.
    *

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCacheMetrics.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.utils;
+
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+/**
+ * Metrics for the usage of ContainerDB.
+ */
+public final class ContainerCacheMetrics {
+
+  private final String name;
+  private final MetricsSystem ms;
+
+  @Metric("Rate to measure the db open latency")
+  private MutableRate dbOpenLatency;
+
+  @Metric("Rate to measure the db close latency")
+  private MutableRate dbCloseLatency;
+
+  @Metric("Number of Container Cache Hits")
+  private MutableCounterLong numCacheHits;
+
+  @Metric("Number of Container Cache Misses")
+  private MutableCounterLong numCacheMisses;
+
+  @Metric("Number of DB.get Ops")
+  private MutableCounterLong numDbGetOps;
+
+  @Metric("Number of DB.remove Ops")
+  private MutableCounterLong numDbRemoveOps;
+
+  @Metric("Number of Container Cache Evictions")
+  private MutableCounterLong numCacheEvictions;
+
+  private ContainerCacheMetrics(String name, MetricsSystem ms) {
+    this.name = name;
+    this.ms = ms;
+  }
+
+  public static ContainerCacheMetrics create() {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    String name = "ContainerCacheMetrics";
+
+    return ms.register(name, "null", new ContainerCacheMetrics(name, ms));
+  }
+
+  public void incNumDbGetOps() {
+    numDbGetOps.incr();
+  }
+
+  public void incNumDbRemoveOps() {
+    numDbRemoveOps.incr();
+  }
+
+  public void incNumCacheMisses() {
+    numCacheMisses.incr();
+  }
+
+  public void incNumCacheHits() {
+    numCacheHits.incr();
+  }
+
+  public void incNumCacheEvictions() {
+    numCacheEvictions.incr();
+  }
+
+  public void incDbCloseLatency(long millis) {
+    dbCloseLatency.add(millis);
+  }
+
+  public void incDbOpenLatency(long millis) {
+    dbOpenLatency.add(millis);
+  }
+
+  public long getNumDbGetOps() {
+    return numDbGetOps.value();
+  }
+
+  public long getNumDbRemoveOps() {
+    return numDbRemoveOps.value();
+  }
+
+  public long getNumCacheMisses() {
+    return numCacheMisses.value();
+  }
+
+  public long getNumCacheHits() {
+    return numCacheHits.value();
+  }
+
+  public long getNumCacheEvictions() {
+    return numCacheEvictions.value();
+  }
+}

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
+import org.apache.hadoop.ozone.container.common.utils.ContainerCacheMetrics;
 import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
 import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
 import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl;
@@ -84,15 +85,22 @@ public void testContainerCacheEviction() throws Exception {
     createContainerDB(conf, containerDir3);
     createContainerDB(conf, containerDir4);
 
+    ContainerCacheMetrics metrics = cache.getMetrics();
+    long numDbGetCount = metrics.getNumDbGetOps();
+    long numCacheMisses = metrics.getNumCacheMisses();
     // Get 2 references out of the same db and verify the objects are same.
     ReferenceCountedDB db1 = cache.getDB(1, "RocksDB",
             containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
     Assert.assertEquals(1, db1.getReferenceCount());
+    Assert.assertEquals(numDbGetCount + 1, metrics.getNumDbGetOps());
     ReferenceCountedDB db2 = cache.getDB(1, "RocksDB",
             containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf);
     Assert.assertEquals(2, db2.getReferenceCount());
+    Assert.assertEquals(numCacheMisses + 1, metrics.getNumCacheMisses());
     Assert.assertEquals(2, db1.getReferenceCount());
     Assert.assertEquals(db1, db2);
+    Assert.assertEquals(numDbGetCount + 2, metrics.getNumDbGetOps());
+    Assert.assertEquals(numCacheMisses + 1, metrics.getNumCacheMisses());
 
     // add one more references to ContainerCache.
     ReferenceCountedDB db3 = cache.getDB(2, "RocksDB",