Use ClassNameFilter to track uninteresting classes (#9976)

* Replace IsPublicFilter with ClassNameFilter
* Replace NoMatchFilter with ClassNameFilter
* Remove now unused ClassCodeFilter
* Document that NoMatchFilter is now a utility class
* Catch potential security exception when checking file / installing shutdown hook

Author: mcculls

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/ClassCodeFilter.java

@@ -2,6 +2,7 @@
 
 import static net.bytebuddy.matcher.ElementMatchers.not;
 
+import datadog.instrument.utils.ClassNameFilter;
 import datadog.trace.agent.tooling.InstrumenterMetrics;
 import datadog.trace.agent.tooling.bytebuddy.TypeInfoCache;
 import datadog.trace.agent.tooling.bytebuddy.TypeInfoCache.SharedTypeInfo;
@@ -61,7 +62,7 @@ enum MatcherKind {
   private static final boolean namesAreUnique = InstrumenterConfig.get().isResolverNamesAreUnique();
 
   // compact filter recording uninteresting types
-  private static final NoMatchFilter noMatchFilter = new NoMatchFilter();
+  private static final ClassNameFilter noMatchFilter = NoMatchFilter.build();
 
   // caches positive memoized matches
   private static final TypeInfoCache<BitSet> memos =

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/memoize/Memoizer.java

@@ -2,7 +2,7 @@
 
 import static datadog.trace.util.AgentThreadFactory.AGENT_THREAD_GROUP;
 
-import datadog.trace.agent.tooling.bytebuddy.ClassCodeFilter;
+import datadog.instrument.utils.ClassNameFilter;
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTraceApiInfo;
 import datadog.trace.api.InstrumenterConfig;
@@ -19,21 +19,37 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/** Compact filter that records uninteresting types. */
-final class NoMatchFilter extends ClassCodeFilter {
+/** Builds a persistable compact filter that records uninteresting types. */
+final class NoMatchFilter {
   private static final Logger log = LoggerFactory.getLogger(NoMatchFilter.class);
 
-  NoMatchFilter() {
-    super(InstrumenterConfig.get().getResolverNoMatchesSize());
+  private static final String TRACER_VERSION_HEADER = "dd-java-agent";
+  private static final String NO_MATCH_FILTER_HEADER = "NoMatchFilter";
 
-    // seed filter from previously collected results?
+  private NoMatchFilter() {}
+
+  public static ClassNameFilter build() {
     Path noMatchFile = discoverNoMatchFile();
     if (null != noMatchFile) {
-      seedNoMatchFilter(noMatchFile);
+      // support persisting/restoring no-match results
+      try {
+        if (Files.exists(noMatchFile)) {
+          // restore existing filter with previously collected results
+          return seedNoMatchFilter(noMatchFile);
+        } else {
+          // populate filter from current run and persist at shutdown
+          ClassNameFilter filter = emptyNoMatchFilter();
+          Runtime.getRuntime().addShutdownHook(new ShutdownHook(noMatchFile, filter));
+          return filter;
+        }
+      } catch (Throwable e) {
+        log.debug("Unable to use NoMatchFilter at {}", noMatchFile, e);
+      }
     }
+    return emptyNoMatchFilter();
   }
 
-  static Path discoverNoMatchFile() {
+  private static Path discoverNoMatchFile() {
     String cacheDir = InstrumenterConfig.get().getResolverCacheDir();
     if (null == cacheDir) {
       return null;
@@ -53,51 +69,49 @@ static Path discoverNoMatchFile() {
     return Paths.get(cacheDir, noMatchFilterName);
   }
 
-  void seedNoMatchFilter(Path noMatchFile) {
-    if (!Files.exists(noMatchFile)) {
-      Runtime.getRuntime().addShutdownHook(new ShutdownHook(noMatchFile));
-    } else {
-      log.debug("Seeding NoMatchFilter from {}", noMatchFile);
-      try (DataInputStream in =
-          new DataInputStream(new BufferedInputStream(Files.newInputStream(noMatchFile)))) {
-        while (true) {
-          switch (in.readUTF()) {
-            case "dd-java-agent":
-              expectVersion(in, DDTraceApiInfo.VERSION);
-              break;
-            case "NoMatchFilter":
-              if (in.readInt() != slots.length) {
-                throw new IOException("filter size mismatch");
-              }
-              for (int i = 0; i < slots.length; i++) {
-                slots[i] = in.readLong();
-              }
-              return;
-            default:
-              throw new IOException("unexpected content");
-          }
-        }
-      } catch (IOException e) {
-        if (log.isDebugEnabled()) {
-          log.info("Unable to seed NoMatchFilter from {}", noMatchFile, e);
-        } else {
-          log.info("Unable to seed NoMatchFilter from {}: {}", noMatchFile, e.getMessage());
+  private static ClassNameFilter emptyNoMatchFilter() {
+    return new ClassNameFilter(InstrumenterConfig.get().getResolverNoMatchesSize());
+  }
+
+  private static ClassNameFilter seedNoMatchFilter(Path noMatchFile) {
+    log.debug("Seeding NoMatchFilter from {}", noMatchFile);
+    try (DataInputStream in =
+        new DataInputStream(new BufferedInputStream(Files.newInputStream(noMatchFile)))) {
+      while (true) {
+        switch (in.readUTF()) {
+          case TRACER_VERSION_HEADER:
+            expectVersion(in, DDTraceApiInfo.VERSION);
+            break;
+          case NO_MATCH_FILTER_HEADER:
+            return ClassNameFilter.readFrom(in);
+          default:
+            throw new IOException("unexpected content");
         }
       }
+    } catch (Throwable e) {
+      if (log.isDebugEnabled()) {
+        log.info("Unable to seed NoMatchFilter from {}", noMatchFile, e);
+      } else {
+        log.info("Unable to seed NoMatchFilter from {}: {}", noMatchFile, e.getMessage());
+      }
+      return emptyNoMatchFilter();
     }
   }
 
-  void persistNoMatchFilter(Path noMatchFile) {
+  private static void expectVersion(DataInputStream in, String version) throws IOException {
+    if (!version.equals(in.readUTF())) {
+      throw new IOException("version mismatch");
+    }
+  }
+
+  static void persistNoMatchFilter(Path noMatchFile, ClassNameFilter filter) {
     log.debug("Persisting NoMatchFilter to {}", noMatchFile);
     try (DataOutputStream out =
         new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(noMatchFile)))) {
-      out.writeUTF("dd-java-agent");
+      out.writeUTF(TRACER_VERSION_HEADER);
       out.writeUTF(DDTraceApiInfo.VERSION);
-      out.writeUTF("NoMatchFilter");
-      out.writeInt(slots.length);
-      for (long slot : slots) {
-        out.writeLong(slot);
-      }
+      out.writeUTF(NO_MATCH_FILTER_HEADER);
+      filter.writeTo(out);
     } catch (IOException e) {
       if (log.isDebugEnabled()) {
         log.info("Unable to persist NoMatchFilter to {}", noMatchFile, e);
@@ -107,23 +121,19 @@ void persistNoMatchFilter(Path noMatchFile) {
     }
   }
 
-  static void expectVersion(DataInputStream in, String version) throws IOException {
-    if (!version.equals(in.readUTF())) {
-      throw new IOException("version mismatch");
-    }
-  }
-
-  class ShutdownHook extends Thread {
+  private static final class ShutdownHook extends Thread {
     private final Path noMatchFile;
+    private final ClassNameFilter filter;
 
-    ShutdownHook(Path noMatchFile) {
+    ShutdownHook(Path noMatchFile, ClassNameFilter filter) {
       super(AGENT_THREAD_GROUP, "dd-NoMatchFilter-persist-hook");
       this.noMatchFile = noMatchFile;
+      this.filter = filter;
     }
 
     @Override
     public void run() {
-      persistNoMatchFilter(noMatchFile);
+      persistNoMatchFilter(noMatchFile, filter);
     }
   }
 }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/memoize/NoMatchFilter.java

@@ -5,6 +5,7 @@
 import static datadog.trace.bootstrap.AgentClassLoading.LOCATING_CLASS;
 import static net.bytebuddy.dynamic.loading.ClassLoadingStrategy.BOOTSTRAP_LOADER;
 
+import datadog.instrument.utils.ClassNameFilter;
 import datadog.trace.agent.tooling.InstrumenterMetrics;
 import datadog.trace.agent.tooling.bytebuddy.ClassFileLocators;
 import datadog.trace.agent.tooling.bytebuddy.TypeInfoCache;
@@ -86,7 +87,8 @@ final class TypeFactory {
   private static final TypeInfoCache<TypeDescription> fullTypes =
       new TypeInfoCache<>(InstrumenterConfig.get().getResolverTypePoolSize());
 
-  static final IsPublicFilter isPublicFilter = new IsPublicFilter();
+  static final ClassNameFilter isPublicFilter =
+      new ClassNameFilter(InstrumenterConfig.get().getResolverVisibilitySize());
 
   /** Small local cache to help deduplicate lookups when matching/transforming. */
   private final DDCache<String, LazyType> deferredTypes = DDCaches.newFixedSizeCache(16);