HDDS-8774. Log allocation stack trace for leaked CodecBuffer

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -19,6 +19,9 @@
 package org.apache.hadoop.hdds;
 
 import com.google.protobuf.ServiceException;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.management.ObjectName;
 import java.io.File;
 import java.io.IOException;
@@ -845,4 +848,26 @@ public static <T, E extends IOException> T preserveThreadName(
       }
     }
   }
+
+  /** Concatenate stack trace {@code elements} (one per line) starting at
+   * {@code startIndex}. */
+  public static @Nonnull String formatStackTrace(
+      @Nullable StackTraceElement[] elements, int startIndex) {
+    if (elements != null && elements.length > startIndex) {
+      final StringBuilder sb = new StringBuilder();
+      for (int line = startIndex; line < elements.length; line++) {
+        sb.append(elements[line]).append("\n");
+      }
+      return sb.toString();
+    }
+    return "";
+  }
+
+  /** @return current thread stack trace if {@code logger} has debug enabled */
+  public static @Nullable StackTraceElement[] getStackTrace(
+      @Nonnull Logger logger) {
+    return logger.isDebugEnabled()
+        ? Thread.currentThread().getStackTrace()
+        : null;
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java

@@ -37,6 +37,9 @@
 import java.util.function.IntFunction;
 import java.util.function.ToIntFunction;
 
+import static org.apache.hadoop.hdds.HddsUtils.formatStackTrace;
+import static org.apache.hadoop.hdds.HddsUtils.getStackTrace;
+
 /**
  * A buffer used by {@link Codec}
  * for supporting RocksDB direct {@link ByteBuffer} APIs.
@@ -53,6 +56,8 @@ public final class CodecBuffer implements AutoCloseable {
       ? POOL.heapBuffer(c, c)   // allocate exact size
       : POOL.heapBuffer(-c);    // allocate a resizable buffer
 
+  private final StackTraceElement[] elements;
+
   /**
    * Allocate a buffer using the given allocator.
    *
@@ -105,6 +110,7 @@ public static void assertNoLeaks() {
 
   private CodecBuffer(ByteBuf buf) {
     this.buf = buf;
+    this.elements = getStackTrace(LOG);
     assertRefCnt(1);
   }
 
@@ -120,8 +126,11 @@ protected void finalize() throws Throwable {
       final int refCnt = buf.refCnt();
       if (refCnt > 0) {
         final int leak = LEAK_COUNT.incrementAndGet();
-        LOG.warn("LEAK {}: {}, refCnt={}, capacity={}",
-            leak, this, refCnt, capacity);
+        LOG.warn("LEAK {}: {}, refCnt={}, capacity={}{}",
+            leak, this, refCnt, capacity,
+            elements != null
+                ? " allocation:\n" + formatStackTrace(elements, 3)
+                : "");
         buf.release(refCnt);
       }
     }

hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java

@@ -20,6 +20,8 @@
 
 import org.rocksdb.RocksObject;
 
+import static org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils.formatStackTrace;
+
 /**
  * General template for a managed RocksObject.
  * @param <T>
@@ -31,11 +33,7 @@ class ManagedObject<T extends RocksObject> implements AutoCloseable {
 
   ManagedObject(T original) {
     this.original = original;
-    if (ManagedRocksObjectUtils.LOG.isDebugEnabled()) {
-      this.elements = Thread.currentThread().getStackTrace();
-    } else {
-      this.elements = null;
-    }
+    this.elements = ManagedRocksObjectUtils.getStackTrace();
   }
 
   public T get() {
@@ -54,15 +52,7 @@ protected void finalize() throws Throwable {
   }
 
   public String getStackTrace() {
-    if (elements != null && elements.length > 0) {
-      StringBuilder sb = new StringBuilder();
-      for (int line = 1; line < elements.length; line++) {
-        sb.append(elements[line]);
-        sb.append("\n");
-      }
-      return sb.toString();
-    }
-    return "";
+    return formatStackTrace(elements);
   }
 
 }

hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java

@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hdds.utils.db.managed;
 
+import org.apache.hadoop.hdds.HddsUtils;
 import org.rocksdb.RocksObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,10 +49,19 @@ static void assertClosed(RocksObject rocksObject, String stackTrace) {
           rocksObject.getClass().getSimpleName());
       if (stackTrace != null && LOG.isDebugEnabled()) {
         String debugMessage = String
-            .format("%n StackTrace for unclosed instance: %s", stackTrace);
+            .format("%nStackTrace for unclosed instance: %s", stackTrace);
         warning = warning.concat(debugMessage);
       }
       LOG.warn(warning);
     }
   }
+
+  static StackTraceElement[] getStackTrace() {
+    return HddsUtils.getStackTrace(LOG);
+  }
+
+  static String formatStackTrace(StackTraceElement[] elements) {
+    return HddsUtils.formatStackTrace(elements, 3);
+  }
+
 }

hadoop-ozone/integration-test/src/test/resources/log4j.properties

@@ -20,3 +20,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%
 log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
 log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
 log4j.logger.org.apache.hadoop.hdds.utils.db.managed=TRACE
+log4j.logger.org.apache.hadoop.hdds.utils.db.CodecBuffer=DEBUG