diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java index df5db9fa9a5f..506fb31f0cb7 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java @@ -22,6 +22,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.Striped; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -33,6 +34,7 @@ import org.apache.hadoop.ozone.container.metadata.DatanodeStore; import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaOneImpl; import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +48,7 @@ public final class ContainerCache extends LRUMap { private static ContainerCache cache; private static final float LOAD_FACTOR = 0.75f; private final Striped rocksDBLock; + private static ContainerCacheMetrics metrics; /** * Constructs a cache that holds DBHandle references. */ @@ -55,6 +58,11 @@ private ContainerCache(int maxSize, int stripes, float loadFactor, boolean rocksDBLock = Striped.lazyWeakLock(stripes); } + @VisibleForTesting + public ContainerCacheMetrics getMetrics() { + return metrics; + } + /** * Return a singleton instance of {@link ContainerCache} * that holds the DB handlers. @@ -71,6 +79,7 @@ public synchronized static ContainerCache getInstance( OzoneConfigKeys.OZONE_CONTAINER_CACHE_LOCK_STRIPES, OzoneConfigKeys.OZONE_CONTAINER_CACHE_LOCK_STRIPES_DEFAULT); cache = new ContainerCache(cacheSize, stripes, LOAD_FACTOR, true); + metrics = ContainerCacheMetrics.create(); } return cache; } @@ -86,7 +95,7 @@ public void shutdownCache() { while (iterator.hasNext()) { iterator.next(); ReferenceCountedDB db = (ReferenceCountedDB) iterator.getValue(); - Preconditions.checkArgument(db.cleanup(), "refCount:", + Preconditions.checkArgument(cleanupDb(db), "refCount:", db.getReferenceCount()); } // reset the cache @@ -104,7 +113,8 @@ protected boolean removeLRU(LinkEntry entry) { ReferenceCountedDB db = (ReferenceCountedDB) entry.getValue(); lock.lock(); try { - return db.cleanup(); + metrics.incNumCacheEvictions(); + return cleanupDb(db); } finally { lock.unlock(); } @@ -130,19 +140,24 @@ public ReferenceCountedDB getDB(long containerID, String containerDBType, ReferenceCountedDB db; Lock containerLock = rocksDBLock.get(containerDBPath); containerLock.lock(); + metrics.incNumDbGetOps(); try { lock.lock(); try { db = (ReferenceCountedDB) this.get(containerDBPath); if (db != null) { + metrics.incNumCacheHits(); db.incrementReference(); return db; + } else { + metrics.incNumCacheMisses(); } } finally { lock.unlock(); } try { + long start = Time.monotonicNow(); DatanodeStore store; if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) { @@ -157,6 +172,7 @@ public ReferenceCountedDB getDB(long containerID, String containerDBType, } db = new ReferenceCountedDB(store, containerDBPath); + metrics.incDbOpenLatency(Time.monotonicNow() - start); } catch (Exception e) { LOG.error("Error opening DB. Container:{} ContainerPath:{}", containerID, containerDBPath, e); @@ -171,7 +187,7 @@ public ReferenceCountedDB getDB(long containerID, String containerDBType, // increment the reference before returning the object currentDB.incrementReference(); // clean the db created in previous step - db.cleanup(); + cleanupDb(db); return currentDB; } else { this.put(containerDBPath, db); @@ -197,7 +213,7 @@ public void removeDB(String containerDBPath) { try { ReferenceCountedDB db = (ReferenceCountedDB)this.get(containerDBPath); if (db != null) { - Preconditions.checkArgument(db.cleanup(), "refCount:", + Preconditions.checkArgument(cleanupDb(db), "refCount:", db.getReferenceCount()); } this.remove(containerDBPath); @@ -206,6 +222,15 @@ public void removeDB(String containerDBPath) { } } + private boolean cleanupDb(ReferenceCountedDB db) { + long time = Time.monotonicNow(); + boolean ret = db.cleanup(); + if (ret) { + metrics.incDbCloseLatency(Time.monotonicNow() - time); + } + return ret; + } + /** * Add a DB handler into cache. * diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCacheMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCacheMetrics.java new file mode 100644 index 000000000000..70f8225af57e --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCacheMetrics.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.common.utils; + +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableRate; + +/** + * Metrics for the usage of ContainerDB. + */ +public final class ContainerCacheMetrics { + + private final String name; + private final MetricsSystem ms; + + @Metric("Rate to measure the db open latency") + private MutableRate dbOpenLatency; + + @Metric("Rate to measure the db close latency") + private MutableRate dbCloseLatency; + + @Metric("Number of Container Cache Hits") + private MutableCounterLong numCacheHits; + + @Metric("Number of Container Cache Misses") + private MutableCounterLong numCacheMisses; + + @Metric("Number of DB.get Ops") + private MutableCounterLong numDbGetOps; + + @Metric("Number of DB.remove Ops") + private MutableCounterLong numDbRemoveOps; + + @Metric("Number of Container Cache Evictions") + private MutableCounterLong numCacheEvictions; + + private ContainerCacheMetrics(String name, MetricsSystem ms) { + this.name = name; + this.ms = ms; + } + + public static ContainerCacheMetrics create() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + String name = "ContainerCacheMetrics"; + + return ms.register(name, "null", new ContainerCacheMetrics(name, ms)); + } + + public void incNumDbGetOps() { + numDbGetOps.incr(); + } + + public void incNumDbRemoveOps() { + numDbRemoveOps.incr(); + } + + public void incNumCacheMisses() { + numCacheMisses.incr(); + } + + public void incNumCacheHits() { + numCacheHits.incr(); + } + + public void incNumCacheEvictions() { + numCacheEvictions.incr(); + } + + public void incDbCloseLatency(long millis) { + dbCloseLatency.add(millis); + } + + public void incDbOpenLatency(long millis) { + dbOpenLatency.add(millis); + } + + public long getNumDbGetOps() { + return numDbGetOps.value(); + } + + public long getNumDbRemoveOps() { + return numDbRemoveOps.value(); + } + + public long getNumCacheMisses() { + return numCacheMisses.value(); + } + + public long getNumCacheHits() { + return numCacheHits.value(); + } + + public long getNumCacheEvictions() { + return numCacheEvictions.value(); + } +} diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java index c00bab5ae777..3a47120181ff 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerCache.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.utils.ContainerCache; +import org.apache.hadoop.ozone.container.common.utils.ContainerCacheMetrics; import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB; import org.apache.hadoop.ozone.container.metadata.DatanodeStore; import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl; @@ -84,15 +85,22 @@ public void testContainerCacheEviction() throws Exception { createContainerDB(conf, containerDir3); createContainerDB(conf, containerDir4); + ContainerCacheMetrics metrics = cache.getMetrics(); + long numDbGetCount = metrics.getNumDbGetOps(); + long numCacheMisses = metrics.getNumCacheMisses(); // Get 2 references out of the same db and verify the objects are same. ReferenceCountedDB db1 = cache.getDB(1, "RocksDB", containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf); Assert.assertEquals(1, db1.getReferenceCount()); + Assert.assertEquals(numDbGetCount + 1, metrics.getNumDbGetOps()); ReferenceCountedDB db2 = cache.getDB(1, "RocksDB", containerDir1.getPath(), OzoneConsts.SCHEMA_LATEST, conf); Assert.assertEquals(2, db2.getReferenceCount()); + Assert.assertEquals(numCacheMisses + 1, metrics.getNumCacheMisses()); Assert.assertEquals(2, db1.getReferenceCount()); Assert.assertEquals(db1, db2); + Assert.assertEquals(numDbGetCount + 2, metrics.getNumDbGetOps()); + Assert.assertEquals(numCacheMisses + 1, metrics.getNumCacheMisses()); // add one more references to ContainerCache. ReferenceCountedDB db3 = cache.getDB(2, "RocksDB",