Extended appsec request body collection (#8748)

What Does This Do
Adds the APPSEC_RASP_COLLECT_REQUEST_BODY flag, which enables collection of request body. This feature is disabled by default.

if APPSEC_RASP_COLLECT_REQUEST_BODY is enabled and there is a RASP event put the same parsed request body that is sent to the WAF via meta_struct with http.request.body key

Add listener to ObjectInstrospection#convert to add boolean tag _dd.appsec.rasp.request_body_size.exceeded if a limit is surpassed

Author: jandro996

dd-java-agent/appsec/src/main/java/com/datadog/appsec/ddwaf/WAFModule.java

@@ -480,6 +480,7 @@ public void onDataAvailable(
         }
 
         if (gwCtx.isRasp) {
+          reqCtx.setRaspMatched(true);
           WafMetricCollector.get().raspRuleMatch(gwCtx.raspRuleType);
         }
 

dd-java-agent/appsec/src/main/java/com/datadog/appsec/event/data/ObjectIntrospection.java

@@ -38,6 +38,16 @@ public final class ObjectIntrospection {
 
   private ObjectIntrospection() {}
 
+  /**
+   * Listener interface for optional per-call truncation logic. Single-method invoked when any
+   * truncation occurs, receiving only the request context.
+   */
+  @FunctionalInterface
+  public interface TruncationListener {
+    /** Called after default truncation handling if any truncation occurred. */
+    void onTruncation();
+  }
+
   /**
    * Converts arbitrary objects compatible with ddwaf_object. Possible types in the result are:
    *
@@ -68,12 +78,26 @@ private ObjectIntrospection() {}
    * @return the converted object
    */
   public static Object convert(Object obj, AppSecRequestContext requestContext) {
+    return convert(obj, requestContext, null);
+  }
+
+  /**
+   * Core conversion method with an optional per-call truncation listener. Always applies default
+   * truncation logic, then invokes listener if provided.
+   */
+  public static Object convert(
+      Object obj, AppSecRequestContext requestContext, TruncationListener listener) {
     State state = new State(requestContext);
     Object converted = guardedConversion(obj, 0, state);
     if (state.stringTooLong || state.listMapTooLarge || state.objectTooDeep) {
+      // Default truncation handling: always run
       requestContext.setWafTruncated();
       WafMetricCollector.get()
           .wafInputTruncated(state.stringTooLong, state.listMapTooLarge, state.objectTooDeep);
+      // Optional extra per-call logic: only requestContext is passed
+      if (listener != null) {
+        listener.onTruncation();
+      }
     }
     return converted;
   }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java

@@ -129,6 +129,9 @@ public class AppSecRequestContext implements DataBundle, Closeable {
   private volatile int wafTimeouts;
   private volatile int raspTimeouts;
 
+  private volatile Object processedRequestBody;
+  private volatile boolean raspMatched;
+
   // keep a reference to the last published usr.id
   private volatile String userId;
   // keep a reference to the last published usr.login
@@ -675,4 +678,20 @@ public boolean isWafContextClosed() {
   void setRequestEndCalled() {
     requestEndCalled = true;
   }
+
+  public void setProcessedRequestBody(Object processedRequestBody) {
+    this.processedRequestBody = processedRequestBody;
+  }
+
+  public Object getProcessedRequestBody() {
+    return processedRequestBody;
+  }
+
+  public boolean isRaspMatched() {
+    return raspMatched;
+  }
+
+  public void setRaspMatched(boolean raspMatched) {
+    this.raspMatched = raspMatched;
+  }
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java

@@ -77,6 +77,7 @@ public class GatewayBridge {
   private static final String USER_COLLECTION_MODE_TAG = "_dd.appsec.user.collection_mode";
 
   private static final Map<LoginEvent, Address<?>> EVENT_MAPPINGS = new EnumMap<>(LoginEvent.class);
+  private static final String METASTRUCT_REQUEST_BODY = "http.request.body";
 
   static {
     EVENT_MAPPINGS.put(LoginEvent.LOGIN_SUCCESS, KnownAddresses.LOGIN_SUCCESS);
@@ -572,9 +573,20 @@ private Flow<Void> onRequestBodyProcessed(RequestContext ctx_, Object obj) {
       if (subInfo == null || subInfo.isEmpty()) {
         return NoopFlow.INSTANCE;
       }
-      DataBundle bundle =
-          new SingletonDataBundle<>(
-              KnownAddresses.REQUEST_BODY_OBJECT, ObjectIntrospection.convert(obj, ctx));
+      Object converted =
+          ObjectIntrospection.convert(
+              obj,
+              ctx,
+              () -> {
+                if (Config.get().isAppSecRaspCollectRequestBody()) {
+                  ctx_.getTraceSegment()
+                      .setTagTop("_dd.appsec.rasp.request_body_size.exceeded", true);
+                }
+              });
+      if (Config.get().isAppSecRaspCollectRequestBody()) {
+        ctx.setProcessedRequestBody(converted);
+      }
+      DataBundle bundle = new SingletonDataBundle<>(KnownAddresses.REQUEST_BODY_OBJECT, converted);
       try {
         GatewayContext gwCtx = new GatewayContext(false);
         return producerService.publishDataEvent(subInfo, ctx, bundle, gwCtx);
@@ -723,6 +735,12 @@ private NoopFlow onRequestEnded(RequestContext ctx_, IGSpanInfo spanInfo) {
           StackUtils.addStacktraceEventsToMetaStruct(ctx_, METASTRUCT_EXPLOIT, stackTraces);
         }
 
+        // Report collected parsed request body if there is a RASP event
+        if (ctx.isRaspMatched() && ctx.getProcessedRequestBody() != null) {
+          ctx_.getOrCreateMetaStructTop(
+              METASTRUCT_REQUEST_BODY, k -> ctx.getProcessedRequestBody());
+        }
+
       } else if (hasUserInfo(traceSeg)) {
         // Report all collected request headers on user tracking event
         writeRequestHeaders(traceSeg, REQUEST_HEADERS_ALLOW_LIST, ctx.getRequestHeaders(), false);

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/event/data/ObjectIntrospectionSpecification.groovy

@@ -304,4 +304,18 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     expect:
     convert([cs], ctx) == ['error:my exception']
   }
+
+  void 'truncated conversion triggers truncation listener if available '() {
+    setup:
+    def listener = Mock(ObjectIntrospection.TruncationListener)
+    def object = 'A' * 5000
+
+    when:
+    convert(object, ctx, listener)
+
+    then:
+    1 * ctx.setWafTruncated()
+    1 * wafMetricCollector.wafInputTruncated(true, false, false)
+    1 * listener.onTruncation()
+  }
 }

dd-smoke-tests/appsec/springboot/src/test/groovy/datadog/smoketest/appsec/ExtendedDataCollectionSmokeTest.groovy

@@ -1,19 +1,63 @@
 package datadog.smoketest.appsec
 
 import datadog.trace.agent.test.utils.OkHttpUtils
+import okhttp3.FormBody
 import okhttp3.Request
+import spock.lang.Shared
 
 class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
 
+  @Shared
+  String buildDir = new File(System.getProperty("datadog.smoketest.builddir")).absolutePath
+  @Shared
+  String customRulesPath = "${buildDir}/appsec_custom_rules.json"
+
+  def prepareCustomRules() {
+    // Prepare ruleset with additional test rules
+    mergeRules(
+      customRulesPath,
+      [
+        [
+          id          : 'rasp-932-100',  // to replace default rule
+          name        : 'Shell command injection exploit',
+          enable      : 'true',
+          tags        : [
+            type: 'command_injection',
+            category: 'vulnerability_trigger',
+            cwe: '77',
+            capec: '1000/152/248/88',
+            confidence: '0',
+            module: 'rasp'
+          ],
+          conditions  : [
+            [
+              parameters: [
+                resource: [[address: 'server.sys.shell.cmd']],
+                params  : [[address: 'server.request.body']],
+              ],
+              operator  : "shi_detector",
+            ],
+          ],
+          transformers: [],
+          on_match    : ['block']
+        ]
+      ])
+  }
+
   @Override
   ProcessBuilder createProcessBuilder() {
 
+    // We run this here to ensure it runs before starting the process. Child setupSpec runs after parent setupSpec,
+    // so it is not a valid location.
+    prepareCustomRules()
+
     String springBootShadowJar = System.getProperty("datadog.smoketest.appsec.springboot.shadowJar.path")
 
     List<String> command = new ArrayList<>()
     command.add(javaPath())
     command.addAll(defaultJavaProperties)
     command.addAll(defaultAppSecProperties)
+    command.add('-Ddd.appsec.rasp.collect.request.body=true')
     command.add('-Ddd.appsec.collect.all.headers=true')
     command.add('-Ddd.appsec.header.collection.redaction.enabled=false')
     command.addAll((String[]) ["-jar", springBootShadowJar, "--server.port=${httpPort}"])
@@ -146,6 +190,124 @@ class ExtendedDataCollectionSmokeTest extends AbstractAppSecServerSmokeTest {
     rootSpan.meta.get('http.response.headers.content-language') == 'en-US'
   }
 
+  void 'test request body collection if RASP event'(){
+    when:
+    String url = "http://localhost:${httpPort}/shi/cmd"
+    def formBuilder = new FormBody.Builder()
+    formBuilder.add('cmd', '$(cat /etc/passwd 1>&2 ; echo .)')
+    final body = formBuilder.build()
+    def request = new Request.Builder()
+      .url(url)
+      .post(body)
+      .build()
+    def response = client.newCall(request).execute()
+    def responseBodyStr = response.body().string()
+
+    then:
+    response.code() == 403
+    responseBodyStr.contains('You\'ve been blocked')
+
+    when:
+    waitForTraceCount(1)
+
+    then:
+    def rootSpans = this.rootSpans.toList()
+    rootSpans.size() == 1
+    def rootSpan = rootSpans[0]
+
+    def trigger = null
+    for (t in rootSpan.triggers) {
+      if (t['rule']['id'] == 'rasp-932-100') {
+        trigger = t
+        break
+      }
+    }
+    assert trigger != null, 'test trigger not found'
+
+    rootSpan.span.metaStruct != null
+    def requestBody = rootSpan.span.metaStruct.get('http.request.body')
+    assert requestBody != null, 'request body is not set'
+    !rootSpan.meta.containsKey('_dd.appsec.rasp.request_body_size.exceeded')
+
+  }
+
+  void 'test request body collection if RASP event exceeded'(){
+    when:
+    String url = "http://localhost:${httpPort}/shi/cmd"
+    def formBuilder = new FormBody.Builder()
+    formBuilder.add('cmd', '$(cat /etc/passwd 1>&2 ; echo .)'+'A' * 5000)
+    final body = formBuilder.build()
+    def request = new Request.Builder()
+      .url(url)
+      .post(body)
+      .build()
+    def response = client.newCall(request).execute()
+    def responseBodyStr = response.body().string()
+
+    then:
+    response.code() == 403
+    responseBodyStr.contains('You\'ve been blocked')
+
+    when:
+    waitForTraceCount(1)
+
+    then:
+    def rootSpans = this.rootSpans.toList()
+    rootSpans.size() == 1
+    def rootSpan = rootSpans[0]
+
+    def trigger = null
+    for (t in rootSpan.triggers) {
+      if (t['rule']['id'] == 'rasp-932-100') {
+        trigger = t
+        break
+      }
+    }
+    assert trigger != null, 'test trigger not found'
+
+    rootSpan.span.metaStruct != null
+    def requestBody = rootSpan.span.metaStruct.get('http.request.body')
+    assert requestBody != null, 'request body is not set'
+    rootSpan.meta.containsKey('_dd.appsec.rasp.request_body_size.exceeded')
+
+  }
+
+  void 'test request body not collected if no RASP event'(){
+    when:
+    String url = "http://localhost:${httpPort}/greeting"
+    def formBuilder = new FormBody.Builder()
+    formBuilder.add('cmd', 'test')
+    final body = formBuilder.build()
+    def request = new Request.Builder()
+      .url(url)
+      .post(body)
+      .build()
+    def response = client.newCall(request).execute()
+
+    then:
+    response.code() == 200
+
+    when:
+    waitForTraceCount(1)
+
+    then:
+    def rootSpans = this.rootSpans.toList()
+    rootSpans.size() == 1
+    def rootSpan = rootSpans[0]
+
+    def trigger = null
+    for (t in rootSpan.triggers) {
+      if (t['rule']['id'] == 'rasp-932-100') {
+        trigger = t
+        break
+      }
+    }
+    assert trigger == null, 'test trigger found'
+
+    rootSpan.span.metaStruct == null
+
+  }
+
 
 
 }

dd-smoke-tests/appsec/springboot/src/test/groovy/datadog/smoketest/appsec/SpringBootSmokeTest.groovy

@@ -469,6 +469,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
+    rootSpan.span.metaStruct == null
 
     where:
     variant               | _
@@ -508,6 +509,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
+    rootSpan.span.metaStruct == null
 
     where:
     variant | _
@@ -600,6 +602,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
+    rootSpan.span.metaStruct == null
 
     where:
     endpoint                    | cmd                              | params
@@ -650,6 +653,7 @@ class SpringBootSmokeTest extends AbstractAppSecServerSmokeTest {
       }
     }
     assert trigger != null, 'test trigger not found'
+    rootSpan.span.metaStruct == null
 
     where:
     endpoint           | cmd                                  | params

dd-trace-api/src/main/java/datadog/trace/api/config/AppSecConfig.java

@@ -47,6 +47,7 @@ public final class AppSecConfig {
   public static final String APPSEC_MAX_COLLECTED_HEADERS = "appsec.max.collected.headers";
   public static final String APPSEC_HEADER_COLLECTION_REDACTION_ENABLED =
       "appsec.header.collection.redaction.enabled";
+  public static final String APPSEC_RASP_COLLECT_REQUEST_BODY = "appsec.rasp.collect.request.body";
 
   private AppSecConfig() {}
 }