Skip to content

Add support for filtering shaded third-party libs #8612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 26, 2025
Merged

Add support for filtering shaded third-party libs #8612

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
067bbb7
Add support for filtering shaded third-party libs
jpbempel Mar 21, 2025
d511e34
fix shaded detection
jpbempel Mar 25, 2025
c7e292e
remove unused var
jpbempel Mar 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions ...va-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ThirdPartyLibraries.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -22,6 +24,20 @@ public class ThirdPartyLibraries {
private static final JsonAdapter<InternalConfig> ADAPTER =
new Moshi.Builder().build().adapter(InternalConfig.class);
private static final String FILE_NAME = "/third_party_libraries.json";
private static final Set<String> DEFAULT_SHADING_IDENTIFIERS =
new HashSet<>(
Arrays.asList(
"shaded",
"thirdparty",
"dependencies",
"relocated",
"bundled",
"embedded",
"vendor",
"repackaged",
"shadow",
"shim",
"wrapper"));

private ThirdPartyLibraries() {}

Expand All @@ -46,6 +62,13 @@ public Set<String> getThirdPartyExcludes(Config config) {
.collect(Collectors.toSet());
}

public Set<String> getShadingIdentifiers(Config config) {
Stream<String> configStream =
config.getThirdPartyShadingIdentifiers().stream().filter(s -> !s.isEmpty());
return Stream.concat(configStream, DEFAULT_SHADING_IDENTIFIERS.stream())
.collect(Collectors.toSet());
}

// Add a*, b*, c*, ..., z* to the exclude trie in ClassNameFiltering. Simply adding * does not
// work.
private static Set<String> getExcludeAll() {
Expand Down
3 changes: 2 additions & 1 deletion ...agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.datadog.debugger.symbol;

import datadog.trace.bootstrap.debugger.DebuggerContext.ClassNameFilter;
import datadog.trace.util.Strings;
import java.lang.instrument.ClassFileTransformer;
import java.security.ProtectionDomain;
import org.slf4j.Logger;
Expand Down Expand Up @@ -34,7 +35,7 @@ public byte[] transform(
// Don't parse our own classes to avoid duplicate class definition
return null;
}
if (classNameFiltering.isExcluded(className)) {
if (classNameFiltering.isExcluded(Strings.getClassName(className))) {
return null;
}
symbolAggregator.parseClass(className, classfileBuffer, protectionDomain);
Expand Down
28 changes: 23 additions & 5 deletions dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/ClassNameFiltering.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import datadog.trace.api.Config;
import datadog.trace.bootstrap.debugger.DebuggerContext.ClassNameFilter;
import datadog.trace.util.ClassNameTrie;
import datadog.trace.util.Strings;
import java.util.Collections;
import java.util.Set;
import java.util.regex.Pattern;
Expand All @@ -14,36 +15,53 @@ public class ClassNameFiltering implements ClassNameFilter {

private final ClassNameTrie includeTrie;
private final ClassNameTrie excludeTrie;
private final Set<String> shadingIdentifiers;

public ClassNameFiltering(Config config) {
this(
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config),
ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(config));
ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(config),
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(config));
}

public ClassNameFiltering(Set<String> excludes) {
this(excludes, Collections.emptySet());
this(excludes, Collections.emptySet(), Collections.emptySet());
}

public ClassNameFiltering(Set<String> excludes, Set<String> includes) {
public ClassNameFiltering(
Set<String> excludes, Set<String> includes, Set<String> shadingIdentifiers) {
ClassNameTrie.Builder excludeBuilder = new ClassNameTrie.Builder();
excludes.forEach(s -> excludeBuilder.put(s + "*", 1));
this.excludeTrie = excludeBuilder.buildTrie();
ClassNameTrie.Builder includeBuilder = new ClassNameTrie.Builder();
includes.forEach(s -> includeBuilder.put(s + "*", 1));
this.includeTrie = includeBuilder.buildTrie();
this.shadingIdentifiers = shadingIdentifiers;
}

// className is the fully qualified class name with '.' (Java type) notation
public boolean isExcluded(String className) {
return (includeTrie.apply(className) < 0 && excludeTrie.apply(className) > 0)
|| isLambdaProxyClass(className);
|| isLambdaProxyClass(className)
|| isShaded(className);
Copy link
Member

@ojung ojung Mar 24, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should make sure that we don't exclude first-party shaded classes. E.g. com.example.shaded.com.example.MyClass. Not sure it's a common pattern, but in the backend we're excluding this way.

}

static boolean isLambdaProxyClass(String className) {
return LAMBDA_PROXY_CLASS_PATTERN.matcher(className).matches();
}

boolean isShaded(String className) {
String packageName = Strings.getPackageName(className);
for (String shadingIdentifier : shadingIdentifiers) {
if (packageName.contains(shadingIdentifier)) {
return true;
}
}
return false;
}

public static ClassNameFiltering allowAll() {
return new ClassNameFiltering(Collections.emptySet(), Collections.emptySet());
return new ClassNameFiltering(
Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
}
}
21 changes: 20 additions & 1 deletion ...gent/agent-debugger/src/test/java/com/datadog/debugger/agent/ThirdPartyLibrariesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.datadog.debugger.agent;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand All @@ -20,6 +21,7 @@ class ThirdPartyLibrariesTest {
void setUp() {
when(mockConfig.getThirdPartyIncludes()).thenReturn(Collections.emptySet());
when(mockConfig.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
when(mockConfig.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
}

@Test
Expand All @@ -29,7 +31,6 @@ void testGetExcludesContainsDefaultExclude() {

@Test
void testGetExcludesWithExplicitExclude() {

when(mockConfig.getThirdPartyIncludes())
.thenReturn(Collections.singleton("com.datadog.debugger"));
assertTrue(
Expand Down Expand Up @@ -72,4 +73,22 @@ void testGetExcludeAll() {
Set<String> excludeAll = ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(null);
for (char c : ThirdPartyLibraries.ALPHABET) assertTrue(excludeAll.contains(String.valueOf(c)));
}

@Test
void testEmptyStrings() {
int expectedIncludeDefaultSize =
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(mockConfig).size();
int expectedShadingDefaultSize =
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(mockConfig).size();
when(mockConfig.getThirdPartyIncludes()).thenReturn(Collections.singleton(""));
when(mockConfig.getThirdPartyExcludes()).thenReturn(Collections.singleton(""));
when(mockConfig.getThirdPartyShadingIdentifiers()).thenReturn(Collections.singleton(""));
assertEquals(
expectedIncludeDefaultSize,
ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(mockConfig).size());
assertTrue(ThirdPartyLibraries.INSTANCE.getThirdPartyExcludes(mockConfig).isEmpty());
assertEquals(
expectedShadingDefaultSize,
ThirdPartyLibraries.INSTANCE.getShadingIdentifiers(mockConfig).size());
}
}
3 changes: 2 additions & 1 deletion ...a-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymDBEnablementTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ public void noDuplicateSymbolExtraction() {
ClassNameFiltering classNameFiltering =
new ClassNameFiltering(
Collections.singleton("org.springframework."),
Collections.singleton("com.datadog.debugger."));
Collections.singleton("com.datadog.debugger."),
Collections.emptySet());
SymbolAggregator symbolAggregator = new SymbolAggregator(classNameFiltering, mockSymbolSink, 1);
SymDBEnablement symDBEnablement =
new SymDBEnablement(instr, config, symbolAggregator, classNameFiltering);
Expand Down
3 changes: 2 additions & 1 deletion ...t-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -982,7 +982,8 @@ private SymbolExtractionTransformer createTransformer(
return createTransformer(
symbolSink,
symbolFlushThreshold,
new ClassNameFiltering(TRANSFORMER_EXCLUDES, Collections.singleton(SYMBOL_PACKAGE)));
new ClassNameFiltering(
TRANSFORMER_EXCLUDES, Collections.singleton(SYMBOL_PACKAGE), Collections.emptySet()));
}

private SymbolExtractionTransformer createTransformer(
Expand Down
18 changes: 11 additions & 7 deletions ...-agent/agent-debugger/src/test/java/com/datadog/debugger/util/ClassNameFilteringTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.datadog.debugger.agent.ThirdPartyLibraries;
import datadog.trace.api.Config;
import java.util.Collections;
import java.util.HashSet;
Expand Down Expand Up @@ -52,15 +51,18 @@ public void testIncludeOverridesExclude() {
ClassNameFiltering classNameFiltering =
new ClassNameFiltering(
Collections.singleton("com.datadog.debugger"),
Collections.singleton("com.datadog.debugger"));
Collections.singleton("com.datadog.debugger"),
Collections.emptySet());
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
}

@Test
public void testIncludePrefixOverridesExclude() {
ClassNameFiltering classNameFiltering =
new ClassNameFiltering(
Collections.singleton("com.datadog.debugger"), Collections.singleton("com.datadog"));
Collections.singleton("com.datadog.debugger"),
Collections.singleton("com.datadog"),
Collections.emptySet());
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
}

Expand All @@ -69,7 +71,8 @@ public void testIncludeSomeExcludeSome() {
ClassNameFiltering classNameFiltering =
new ClassNameFiltering(
Stream.of("com.datadog.debugger", "org.junit").collect(Collectors.toSet()),
Collections.singleton("com.datadog.debugger"));
Collections.singleton("com.datadog.debugger"),
Collections.emptySet());
assertFalse(classNameFiltering.isExcluded("com.datadog.debugger.FooBar"));
assertTrue(classNameFiltering.isExcluded("org.junit.FooBar"));
}
Expand All @@ -82,14 +85,15 @@ public void testIncludeSomeExcludeSome() {
"akka.Actor",
"cats.Functor",
"org.junit.jupiter.api.Test",
"org.datadog.jmxfetch.FooBar"
"org.datadog.jmxfetch.FooBar",
"shaded.org.junit.Test"
})
public void testExcludeDefaults(String input) {
Config config = mock(Config.class);
when(config.getThirdPartyExcludes()).thenReturn(Collections.emptySet());
when(config.getThirdPartyIncludes()).thenReturn(Collections.emptySet());
ClassNameFiltering classNameFiltering =
new ClassNameFiltering(ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config));
when(config.getThirdPartyShadingIdentifiers()).thenReturn(Collections.emptySet());
ClassNameFiltering classNameFiltering = new ClassNameFiltering(config);
assertTrue(classNameFiltering.isExcluded(input));
}

Expand Down
1 change: 1 addition & 0 deletions dd-trace-api/src/main/java/datadog/trace/api/config/DebuggerConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public final class DebuggerConfig {
public static final String DISTRIBUTED_DEBUGGER_ENABLED = "distributed.debugger.enabled";
public static final String THIRD_PARTY_INCLUDES = "third.party.includes";
public static final String THIRD_PARTY_EXCLUDES = "third.party.excludes";
public static final String THIRD_PARTY_SHADING_IDENTIFIERS = "third.party.shading.identifiers";

private DebuggerConfig() {}
}
7 changes: 7 additions & 0 deletions internal-api/src/main/java/datadog/trace/api/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,7 @@ public static String getHostName() {

private final Set<String> debuggerThirdPartyIncludes;
private final Set<String> debuggerThirdPartyExcludes;
private final Set<String> debuggerShadingIdentifiers;

private final boolean awsPropagationEnabled;
private final boolean sqsPropagationEnabled;
Expand Down Expand Up @@ -1734,6 +1735,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())

debuggerThirdPartyIncludes = tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_INCLUDES));
debuggerThirdPartyExcludes = tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_EXCLUDES));
debuggerShadingIdentifiers =
tryMakeImmutableSet(configProvider.getList(THIRD_PARTY_SHADING_IDENTIFIERS));

awsPropagationEnabled = isPropagationEnabled(true, "aws", "aws-sdk");
sqsPropagationEnabled = isPropagationEnabled(true, "sqs");
Expand Down Expand Up @@ -3299,6 +3302,10 @@ public Set<String> getThirdPartyExcludes() {
return debuggerThirdPartyExcludes;
}

public Set<String> getThirdPartyShadingIdentifiers() {
return debuggerShadingIdentifiers;
}

private String getFinalDebuggerBaseUrl() {
if (agentUrl.startsWith("unix:")) {
// provide placeholder agent URL, in practice we'll be tunnelling over UDS
Expand Down