Skip to content

Commit 4a7f585

Browse files
jpbempeljpbempel
authored
Make source file tracking asynchronous (#8684)
Source file tracking is enqueueing classfile buffer to be processed by a background thread avoiding startup delay.
1 parent 22535c7 commit 4a7f585

File tree

9 files changed

+135
-17
lines changed

9 files changed

+135
-17
lines changed

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,7 @@ public static synchronized void run(Instrumentation inst, SharedCommunicationObj
7474
Config config = Config.get();
7575
DebuggerContext.initProductConfigUpdater(new DefaultProductConfigUpdater());
7676
classesToRetransformFinder = new ClassesToRetransformFinder();
77-
if (config.isDynamicInstrumentationEnabled() || config.isDebuggerExceptionEnabled()) {
78-
// only activate Source File Tracking if DI or ER is enabled from the start
79-
setupSourceFileTracking(instrumentation, classesToRetransformFinder);
80-
}
77+
setupSourceFileTracking(instrumentation, classesToRetransformFinder);
8178
if (config.isDebuggerCodeOriginEnabled()) {
8279
startCodeOriginForSpans();
8380
}
@@ -314,7 +311,10 @@ private static String getDiagnosticEndpoint(
314311

315312
private static void setupSourceFileTracking(
316313
Instrumentation instrumentation, ClassesToRetransformFinder finder) {
317-
instrumentation.addTransformer(new SourceFileTrackingTransformer(finder));
314+
SourceFileTrackingTransformer sourceFileTrackingTransformer =
315+
new SourceFileTrackingTransformer(finder);
316+
sourceFileTrackingTransformer.start();
317+
instrumentation.addTransformer(sourceFileTrackingTransformer);
318318
}
319319

320320
private static void loadFromFile(

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/SourceFileTrackingTransformer.java‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,57 @@
44
import static com.datadog.debugger.util.ClassFileHelper.stripPackagePath;
55

66
import com.datadog.debugger.util.ClassFileHelper;
7+
import datadog.trace.util.AgentTaskScheduler;
78
import java.lang.instrument.ClassFileTransformer;
89
import java.lang.instrument.IllegalClassFormatException;
910
import java.security.ProtectionDomain;
11+
import java.util.Queue;
12+
import java.util.concurrent.ConcurrentLinkedQueue;
13+
import java.util.concurrent.TimeUnit;
14+
import org.slf4j.Logger;
15+
import org.slf4j.LoggerFactory;
1016

1117
/**
1218
* Permanent Transformer to track all Inner or Top-Level classes associated with the same SourceFile
1319
* (String.java) Allows to get all classes that are dependent from a source file and be able to
1420
* trigger {@link java.lang.instrument.Instrumentation#retransformClasses(Class[])} on them
1521
*/
1622
public class SourceFileTrackingTransformer implements ClassFileTransformer {
23+
private static final Logger LOGGER = LoggerFactory.getLogger(SourceFileTrackingTransformer.class);
24+
1725
private final ClassesToRetransformFinder finder;
26+
private final Queue<SourceFileItem> queue = new ConcurrentLinkedQueue<>();
27+
private final AgentTaskScheduler scheduler = AgentTaskScheduler.INSTANCE;
28+
private AgentTaskScheduler.Scheduled<Runnable> scheduled;
1829

1930
public SourceFileTrackingTransformer(ClassesToRetransformFinder finder) {
2031
this.finder = finder;
2132
}
2233

34+
public void start() {
35+
scheduled = scheduler.scheduleAtFixedRate(this::flush, 0, 1, TimeUnit.SECONDS);
36+
}
37+
38+
public void stop() {
39+
if (scheduled != null) {
40+
scheduled.cancel();
41+
}
42+
}
43+
44+
void flush() {
45+
if (queue.isEmpty()) {
46+
return;
47+
}
48+
int size = queue.size();
49+
long start = System.nanoTime();
50+
SourceFileItem item;
51+
while ((item = queue.poll()) != null) {
52+
registerSourceFile(item.className, item.classfileBuffer);
53+
}
54+
LOGGER.debug(
55+
"flushing {} source file items in {}ms", size, (System.nanoTime() - start) / 1_000_000);
56+
}
57+
2358
@Override
2459
public byte[] transform(
2560
ClassLoader loader,
@@ -31,16 +66,30 @@ public byte[] transform(
3166
if (className == null) {
3267
return null;
3368
}
69+
queue.add(new SourceFileItem(className, classfileBuffer));
70+
return null;
71+
}
72+
73+
private void registerSourceFile(String className, byte[] classfileBuffer) {
3474
String sourceFile = ClassFileHelper.extractSourceFile(classfileBuffer);
3575
if (sourceFile == null) {
36-
return null;
76+
return;
3777
}
3878
String simpleClassName = stripPackagePath(className);
3979
String simpleSourceFile = removeExtension(sourceFile);
4080
if (simpleClassName.equals(simpleSourceFile)) {
41-
return null;
81+
return;
4282
}
4383
finder.register(sourceFile, className);
44-
return null;
84+
}
85+
86+
private static class SourceFileItem {
87+
final String className;
88+
final byte[] classfileBuffer;
89+
90+
public SourceFileItem(String className, byte[] classfileBuffer) {
91+
this.className = className;
92+
this.classfileBuffer = classfileBuffer;
93+
}
4594
}
4695
}

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SourceFileTrackingTransformerTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ void transformTopLevel() throws IllegalClassFormatException {
3232
null,
3333
null,
3434
getClassFileBytes(MyTopLevelClass.class));
35+
sourceFileTrackingTransformer.flush();
3536
changedClasses =
3637
finder.getAllLoadedChangedClasses(
3738
new Class[] {TopLevelHelper.class, MyTopLevelClass.class}, comparer);
@@ -48,6 +49,7 @@ void transformInner() throws IllegalClassFormatException {
4849
ConfigurationComparer comparer = createComparer("InnerHelper.java");
4950
sourceFileTrackingTransformer.transform(
5051
null, getInternalName(InnerHelper.class), null, null, getClassFileBytes(InnerHelper.class));
52+
sourceFileTrackingTransformer.flush();
5153
List<Class<?>> changedClasses =
5254
finder.getAllLoadedChangedClasses(new Class[] {InnerHelper.class}, comparer);
5355
assertEquals(1, changedClasses.size());
@@ -64,6 +66,7 @@ void transformInner() throws IllegalClassFormatException {
6466
null,
6567
null,
6668
getClassFileBytes(InnerHelper.MySecondInner.class));
69+
sourceFileTrackingTransformer.flush();
6770
changedClasses =
6871
finder.getAllLoadedChangedClasses(
6972
new Class[] {

‎dd-smoke-tests/debugger-integration-tests/src/main/java/datadog/smoketest/debugger/ServerDebuggerTestApplication.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public static void registerMethods() {
4343
methodsByName.put("tracedMethod", ServerDebuggerTestApplication::runTracedMethod);
4444
methodsByName.put("loopingFullMethod", ServerDebuggerTestApplication::runLoopingFullMethod);
4545
methodsByName.put("loopingTracedMethod", ServerDebuggerTestApplication::runLoopingTracedMethod);
46+
methodsByName.put("topLevelMethod", ServerDebuggerTestApplication::runTopLevelMethod);
4647
}
4748

4849
public ServerDebuggerTestApplication(String controlServerUrl) {
@@ -163,6 +164,10 @@ private static void runLoopingTracedMethod(String arg) {
163164
}
164165
}
165166

167+
private static String runTopLevelMethod(String arg) {
168+
return TopLevel.process(arg);
169+
}
170+
166171
private static String fullMethod(
167172
int argInt, String argStr, double argDouble, Map<String, String> argMap, String... argVar) {
168173
try {
@@ -290,3 +295,9 @@ public MockResponse dispatch(RecordedRequest request) throws InterruptedExceptio
290295
}
291296
}
292297
}
298+
299+
class TopLevel {
300+
public static String process(String arg) {
301+
return "TopLevel.process: " + arg;
302+
}
303+
}

‎dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/CodeOriginIntegrationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ void testCodeOriginTraceAnnotation() throws Exception {
4545
assertEquals("runTracedMethod", span.getMeta().get(DD_CODE_ORIGIN_FRAME_METHOD));
4646
assertEquals(
4747
"(java.lang.String)", span.getMeta().get(DD_CODE_ORIGIN_FRAME_SIGNATURE));
48-
assertEquals("145", span.getMeta().get(DD_CODE_ORIGIN_FRAME_LINE));
48+
assertEquals("146", span.getMeta().get(DD_CODE_ORIGIN_FRAME_LINE));
4949
codeOrigin.set(true);
5050
}
5151
}

‎dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/InProductEnablementIntegrationTest.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,28 @@ void testDynamicInstrumentationEnablement() throws Exception {
3939
waitForReTransformation(appUrl); // wait for retransformation of removed probe
4040
}
4141

42+
@Test
43+
@DisplayName("testDynamicInstrumentationEnablementWithLineProbe")
44+
void testDynamicInstrumentationEnablementWithLineProbe() throws Exception {
45+
appUrl = startAppAndAndGetUrl();
46+
setConfigOverrides(createConfigOverrides(true, false));
47+
LogProbe probe =
48+
LogProbe.builder()
49+
.probeId(LINE_PROBE_ID1)
50+
.where("ServerDebuggerTestApplication.java", 301)
51+
.build();
52+
setCurrentConfiguration(createConfig(probe));
53+
waitForFeatureStarted(appUrl, "Dynamic Instrumentation");
54+
execute(appUrl, "topLevelMethod", "");
55+
waitForInstrumentation(appUrl, "datadog.smoketest.debugger.TopLevel");
56+
// disable DI
57+
setConfigOverrides(createConfigOverrides(false, false));
58+
waitForFeatureStopped(appUrl, "Dynamic Instrumentation");
59+
waitForReTransformation(
60+
appUrl,
61+
"datadog.smoketest.debugger.TopLevel"); // wait for retransformation of removed probe
62+
}
63+
4264
@Test
4365
@DisplayName("testDynamicInstrumentationEnablementStaticallyDisabled")
4466
void testDynamicInstrumentationEnablementStaticallyDisabled() throws Exception {

‎dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/LogProbesIntegrationTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,33 @@ void testMultiProbes() throws Exception {
250250
assertFalse(logHasErrors(logFilePath, it -> false));
251251
}
252252

253+
@Test
254+
@DisplayName("testLineProbe")
255+
void testLineProbe() throws Exception {
256+
final String METHOD_NAME = "fullMethod";
257+
final String EXPECTED_UPLOADS = "4"; // 3 statuses + 1 snapshot
258+
LogProbe probe =
259+
LogProbe.builder()
260+
.probeId(LINE_PROBE_ID1)
261+
.where("DebuggerTestApplication.java", 88)
262+
.captureSnapshot(true)
263+
.build();
264+
setCurrentConfiguration(createConfig(probe));
265+
targetProcess = createProcessBuilder(logFilePath, METHOD_NAME, EXPECTED_UPLOADS).start();
266+
AtomicBoolean snapshotReceived = new AtomicBoolean();
267+
registerSnapshotListener(
268+
snapshot -> {
269+
assertEquals(LINE_PROBE_ID1.getId(), snapshot.getProbe().getId());
270+
CapturedContext capturedContext = snapshot.getCaptures().getLines().get(88);
271+
assertFullMethodCaptureArgs(capturedContext);
272+
assertNull(capturedContext.getLocals());
273+
assertNull(capturedContext.getCapturedThrowable());
274+
snapshotReceived.set(true);
275+
});
276+
AtomicBoolean statusResult = registerCheckReceivedInstalledEmitting();
277+
processRequests(() -> snapshotReceived.get() && statusResult.get());
278+
}
279+
253280
@Test
254281
@DisplayName("testSamplingSnapshotDefault")
255282
@DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs")

‎dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ServerAppDebuggerIntegrationTest.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ public class ServerAppDebuggerIntegrationTest extends BaseIntegrationTest {
3131
"datadog.smoketest.debugger.ServerDebuggerTestApplication";
3232
protected static final String CONTROL_URL = "/control";
3333
protected static final ProbeId PROBE_ID = new ProbeId("123356536", 0);
34-
protected static final ProbeId PROBE_ID2 = new ProbeId("1233565367", 12);
34+
protected static final ProbeId LINE_PROBE_ID1 =
35+
new ProbeId("beae1817-f3b0-4ea8-a74f-000000000001", 0);
3536
protected static final String TEST_APP_CLASS_NAME = "ServerDebuggerTestApplication";
3637
protected static final String FULL_METHOD_NAME = "fullMethod";
3738
protected static final String TRACED_METHOD_NAME = "tracedMethod";
@@ -106,9 +107,11 @@ protected void execute(String appUrl, String methodName, String arg) throws IOEx
106107
}
107108

108109
protected void waitForInstrumentation(String appUrl) throws Exception {
109-
String url =
110-
String.format(
111-
appUrl + "/waitForInstrumentation?classname=%s", SERVER_DEBUGGER_TEST_APP_CLASS);
110+
waitForInstrumentation(appUrl, SERVER_DEBUGGER_TEST_APP_CLASS);
111+
}
112+
113+
protected void waitForInstrumentation(String appUrl, String className) throws Exception {
114+
String url = String.format(appUrl + "/waitForInstrumentation?classname=%s", className);
112115
LOG.info("waitForInstrumentation with url={}", url);
113116
sendRequest(url);
114117
AtomicBoolean received = new AtomicBoolean();
@@ -136,9 +139,11 @@ protected void waitForAProbeStatus(ProbeStatus.Status status) throws Exception {
136139
}
137140

138141
protected void waitForReTransformation(String appUrl) throws IOException {
139-
String url =
140-
String.format(
141-
appUrl + "/waitForReTransformation?classname=%s", SERVER_DEBUGGER_TEST_APP_CLASS);
142+
waitForReTransformation(appUrl, SERVER_DEBUGGER_TEST_APP_CLASS);
143+
}
144+
145+
protected void waitForReTransformation(String appUrl, String className) throws IOException {
146+
String url = String.format(appUrl + "/waitForReTransformation?classname=%s", className);
142147
sendRequest(url);
143148
LOG.info("re-transformation done");
144149
}

‎dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SimpleAppDebuggerIntegrationTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ public class SimpleAppDebuggerIntegrationTest extends BaseIntegrationTest {
1111
protected static final String DEBUGGER_TEST_APP_CLASS =
1212
"datadog.smoketest.debugger.DebuggerTestApplication";
1313
protected static final ProbeId PROBE_ID = new ProbeId("123356536", 0);
14-
protected static final ProbeId PROBE_ID2 = new ProbeId("1233565368", 12);
14+
protected static final ProbeId LINE_PROBE_ID1 =
15+
new ProbeId("beae1817-f3b0-4ea8-a74f-000000000001", 0);
1516
protected static final String MAIN_CLASS_NAME = "Main";
1617

1718
@Override

0 commit comments

Comments
 (0)