diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java index d5ee5af8c98f..8df6b2082bd6 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java @@ -46,6 +46,8 @@ import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE; import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE; import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_REPLICA_FOUND; + +import org.apache.hadoop.util.CacheMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +70,7 @@ public class XceiverClientManager implements Closeable, XceiverClientFactory { private final ConfigurationSource conf; private final ScmClientConfig clientConfig; private final Cache clientCache; + private final CacheMetrics cacheMetrics; private List caCerts; private static XceiverClientMetrics metrics; @@ -99,6 +102,7 @@ public XceiverClientManager(ConfigurationSource conf, } this.clientCache = CacheBuilder.newBuilder() + .recordStats() .expireAfterAccess(staleThresholdMs, MILLISECONDS) .maximumSize(clientConf.getMaxSize()) .removalListener( @@ -117,6 +121,8 @@ public void onRemoval( topologyAwareRead = conf.getBoolean( OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT); + + cacheMetrics = CacheMetrics.create(clientCache, this); } @VisibleForTesting @@ -278,6 +284,10 @@ public void close() { //closing is done through RemovalListener clientCache.invalidateAll(); clientCache.cleanUp(); + if (LOG.isDebugEnabled()) { + LOG.debug("XceiverClient cache stats: {}", clientCache.stats()); + } + cacheMetrics.unregister(); if (metrics != null) { metrics.unRegister(); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CacheMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CacheMetrics.java index 8f505f7604df..d1c49402d0e6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CacheMetrics.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CacheMetrics.java @@ -25,11 +25,13 @@ import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.ratis.util.JavaUtils; /** * Reusable component that emits cache metrics for a particular cache. */ public final class CacheMetrics implements MetricsSource { + enum CacheMetricsInfo implements MetricsInfo { CacheName("Cache Metrics."), Size("Size of the cache."), @@ -58,20 +60,26 @@ public String description() { public static final String SOURCE_NAME = CacheMetrics.class.getSimpleName(); - public static final String NAME = CacheMetrics.class.getSimpleName(); - private final Cache cache; private final String name; + private final String sourceName; private CacheMetrics(Cache cache, String name) { this.cache = cache; this.name = name; + sourceName = SOURCE_NAME + "-" + name; + } + + public static CacheMetrics create(Cache cache, Object owner) { + final String name = JavaUtils.getClassSimpleName(owner.getClass()) + + "@" + Integer.toHexString(owner.hashCode()); + return create(cache, name); } public static CacheMetrics create(Cache cache, String name) { MetricsSystem ms = DefaultMetricsSystem.instance(); - return ms.register(NAME, "Cache Metrics", - new CacheMetrics(cache, name)); + CacheMetrics source = new CacheMetrics(cache, name); + return ms.register(source.sourceName, "Cache Metrics", source); } @Override @@ -98,6 +106,6 @@ public void getMetrics(MetricsCollector collector, boolean all) { public void unregister() { MetricsSystem ms = DefaultMetricsSystem.instance(); - ms.unregisterSource(NAME); + ms.unregisterSource(sourceName); } } diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java index 4b87ef74cef1..804b3679a61b 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java @@ -60,21 +60,22 @@ public class TestECBlockOutputStreamEntry { .setState(Pipeline.PipelineState.OPEN) .setNodes(nodes) .build(); - XceiverClientManager manager = - new XceiverClientManager(new OzoneConfiguration()); - HashSet clients = new HashSet<>(); - ECBlockOutputStreamEntry entry = new ECBlockOutputStreamEntry.Builder() - .setXceiverClientManager(manager) - .setPipeline(anECPipeline) - .build(); - for (int i = 0; i < nodes.size(); i++) { - clients.add( - manager.acquireClient( - entry.createSingleECBlockPipeline( - anECPipeline, nodes.get(i), i - ))); + try (XceiverClientManager manager = + new XceiverClientManager(new OzoneConfiguration())) { + HashSet clients = new HashSet<>(); + ECBlockOutputStreamEntry entry = new ECBlockOutputStreamEntry.Builder() + .setXceiverClientManager(manager) + .setPipeline(anECPipeline) + .build(); + for (int i = 0; i < nodes.size(); i++) { + clients.add( + manager.acquireClient( + entry.createSingleECBlockPipeline( + anECPipeline, nodes.get(i), i + ))); + } + assertEquals(5, clients.size()); } - assertEquals(5, clients.size()); } @Test @@ -97,23 +98,26 @@ public class TestECBlockOutputStreamEntry { .setState(Pipeline.PipelineState.OPEN) .setNodes(nodes) .build(); - XceiverClientManager manager = - new XceiverClientManager(new OzoneConfiguration()); - HashSet clients = new HashSet<>(); - ECBlockOutputStreamEntry entry = new ECBlockOutputStreamEntry.Builder() - .setXceiverClientManager(manager) - .setPipeline(anECPipeline) - .build(); - for (int i = 0; i < nodes.size(); i++) { - clients.add( - manager.acquireClient( - entry.createSingleECBlockPipeline( - anECPipeline, nodes.get(i), i - ))); + try (XceiverClientManager manager = + new XceiverClientManager(new OzoneConfiguration())) { + HashSet clients = new HashSet<>(); + ECBlockOutputStreamEntry entry = new ECBlockOutputStreamEntry.Builder() + .setXceiverClientManager(manager) + .setPipeline(anECPipeline) + .build(); + for (int i = 0; i < nodes.size(); i++) { + clients.add( + manager.acquireClient( + entry.createSingleECBlockPipeline( + anECPipeline, nodes.get(i), i + ))); + } + assertEquals(3, clients.size()); + assertEquals(1, + clients.stream().filter(c -> c.getRefcount() == 3).count()); + assertEquals(2, + clients.stream().filter(c -> c.getRefcount() == 1).count()); } - assertEquals(3, clients.size()); - assertEquals(1, clients.stream().filter(c -> c.getRefcount() == 3).count()); - assertEquals(2, clients.stream().filter(c -> c.getRefcount() == 1).count()); } private DatanodeDetails aNode(String ip, String hostName, int port) {