From 0da79e1a73ca830c638e232696cc6fa57eb0decd Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 7 Jun 2023 09:26:27 +0200 Subject: [PATCH] HDDS-8774. Log allocation stack trace for leaked CodecBuffer --- .../org/apache/hadoop/hdds/HddsUtils.java | 25 +++++++++++++++++++ .../hadoop/hdds/utils/db/CodecBuffer.java | 13 ++++++++-- .../hdds/utils/db/managed/ManagedObject.java | 18 +++---------- .../db/managed/ManagedRocksObjectUtils.java | 12 ++++++++- .../src/test/resources/log4j.properties | 1 + 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index a24a0b7c05b2..e55b7340d467 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hdds; import com.google.protobuf.ServiceException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.management.ObjectName; import java.io.File; import java.io.IOException; @@ -845,4 +848,26 @@ public static T preserveThreadName( } } } + + /** Concatenate stack trace {@code elements} (one per line) starting at + * {@code startIndex}. */ + public static @Nonnull String formatStackTrace( + @Nullable StackTraceElement[] elements, int startIndex) { + if (elements != null && elements.length > startIndex) { + final StringBuilder sb = new StringBuilder(); + for (int line = startIndex; line < elements.length; line++) { + sb.append(elements[line]).append("\n"); + } + return sb.toString(); + } + return ""; + } + + /** @return current thread stack trace if {@code logger} has debug enabled */ + public static @Nullable StackTraceElement[] getStackTrace( + @Nonnull Logger logger) { + return logger.isDebugEnabled() + ? Thread.currentThread().getStackTrace() + : null; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java index c8550d59c7c8..fd60f7d9dfd6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java @@ -37,6 +37,9 @@ import java.util.function.IntFunction; import java.util.function.ToIntFunction; +import static org.apache.hadoop.hdds.HddsUtils.formatStackTrace; +import static org.apache.hadoop.hdds.HddsUtils.getStackTrace; + /** * A buffer used by {@link Codec} * for supporting RocksDB direct {@link ByteBuffer} APIs. @@ -53,6 +56,8 @@ public final class CodecBuffer implements AutoCloseable { ? POOL.heapBuffer(c, c) // allocate exact size : POOL.heapBuffer(-c); // allocate a resizable buffer + private final StackTraceElement[] elements; + /** * Allocate a buffer using the given allocator. * @@ -105,6 +110,7 @@ public static void assertNoLeaks() { private CodecBuffer(ByteBuf buf) { this.buf = buf; + this.elements = getStackTrace(LOG); assertRefCnt(1); } @@ -120,8 +126,11 @@ protected void finalize() throws Throwable { final int refCnt = buf.refCnt(); if (refCnt > 0) { final int leak = LEAK_COUNT.incrementAndGet(); - LOG.warn("LEAK {}: {}, refCnt={}, capacity={}", - leak, this, refCnt, capacity); + LOG.warn("LEAK {}: {}, refCnt={}, capacity={}{}", + leak, this, refCnt, capacity, + elements != null + ? " allocation:\n" + formatStackTrace(elements, 3) + : ""); buf.release(refCnt); } } diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java index a5da164c8478..8093882a1d50 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java @@ -20,6 +20,8 @@ import org.rocksdb.RocksObject; +import static org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils.formatStackTrace; + /** * General template for a managed RocksObject. * @param @@ -31,11 +33,7 @@ class ManagedObject implements AutoCloseable { ManagedObject(T original) { this.original = original; - if (ManagedRocksObjectUtils.LOG.isDebugEnabled()) { - this.elements = Thread.currentThread().getStackTrace(); - } else { - this.elements = null; - } + this.elements = ManagedRocksObjectUtils.getStackTrace(); } public T get() { @@ -54,15 +52,7 @@ protected void finalize() throws Throwable { } public String getStackTrace() { - if (elements != null && elements.length > 0) { - StringBuilder sb = new StringBuilder(); - for (int line = 1; line < elements.length; line++) { - sb.append(elements[line]); - sb.append("\n"); - } - return sb.toString(); - } - return ""; + return formatStackTrace(elements); } } diff --git a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java index fc6d2bdd6139..3885100f832d 100644 --- a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java +++ b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hdds.utils.db.managed; +import org.apache.hadoop.hdds.HddsUtils; import org.rocksdb.RocksObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,10 +49,19 @@ static void assertClosed(RocksObject rocksObject, String stackTrace) { rocksObject.getClass().getSimpleName()); if (stackTrace != null && LOG.isDebugEnabled()) { String debugMessage = String - .format("%n StackTrace for unclosed instance: %s", stackTrace); + .format("%nStackTrace for unclosed instance: %s", stackTrace); warning = warning.concat(debugMessage); } LOG.warn(warning); } } + + static StackTraceElement[] getStackTrace() { + return HddsUtils.getStackTrace(LOG); + } + + static String formatStackTrace(StackTraceElement[] elements) { + return HddsUtils.formatStackTrace(elements, 3); + } + } diff --git a/hadoop-ozone/integration-test/src/test/resources/log4j.properties b/hadoop-ozone/integration-test/src/test/resources/log4j.properties index 009160888a25..564b729d5fc1 100644 --- a/hadoop-ozone/integration-test/src/test/resources/log4j.properties +++ b/hadoop-ozone/integration-test/src/test/resources/log4j.properties @@ -20,3 +20,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:% log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR log4j.logger.org.apache.hadoop.hdds.utils.db.managed=TRACE +log4j.logger.org.apache.hadoop.hdds.utils.db.CodecBuffer=DEBUG