Allow multivalue multiline header extraction for ApacheHttpClient

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java

@@ -82,10 +82,15 @@ public AgentSpan onRequest(final AgentSpan span, final REQUEST request) {
       String method = method(request);
       span.setTag(Tags.HTTP_METHOD, method);
 
+      System.out.println("In onRequest method in HttpClientDecorator");
       if (CLIENT_TAG_HEADERS) {
+        System.out.println("Tagging client headers...");
         for (Map.Entry<String, String> headerTag :
             traceConfig(span).getRequestHeaderTags().entrySet()) {
+          System.out.println("HEADERTAG: " + traceConfig(span).getRequestHeaderTags().entrySet());
           String headerValue = getRequestHeader(request, headerTag.getKey());
+          System.out.println("headerTag: " + headerTag.getValue());
+          System.out.println("headerValue: " + headerValue);
           if (null != headerValue) {
             span.setTag(headerTag.getValue(), headerValue);
           }

dd-java-agent/instrumentation/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpClientInstrumentationTest.groovy

@@ -36,10 +36,10 @@ abstract class AkkaHttpClientInstrumentationTest extends HttpClientTest {
   abstract CompletionStage<HttpResponse> doRequest(HttpRequest request)
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = HttpRequest.create(uri.toString())
       .withMethod(HttpMethods.lookup(method).get())
-      .addHeaders(headers.collect { RawHeader.create(it.key, it.value) })
+      .addHeaders(headers.collect { RawHeader.create(it[0], it[1]) })
 
     def response
     try {

dd-java-agent/instrumentation/akka-http/akka-http-10.6/src/test/groovy/AkkaHttpClientInstrumentationTest.groovy

@@ -25,10 +25,10 @@ abstract class AkkaHttpClientInstrumentationTest extends HttpClientTest {
   abstract CompletionStage<HttpResponse> doRequest(HttpRequest request)
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = HttpRequest.create(uri.toString())
       .withMethod(HttpMethods.lookup(method).get())
-      .addHeaders(headers.collect { RawHeader.create(it.key, it.value) })
+      .addHeaders(headers.collect { RawHeader.create(it[0], it[1]) })
 
     def response
     try {

dd-java-agent/instrumentation/apache-httpasyncclient-4/src/main/java/datadog/trace/instrumentation/apachehttpasyncclient/ApacheHttpAsyncClientDecorator.java

@@ -60,9 +60,16 @@ protected int status(final HttpContext context) {
 
   @Override
   protected String getRequestHeader(HttpUriRequest request, String headerName) {
-    Header header = request.getFirstHeader(headerName);
-    if (header != null) {
-      return header.getValue();
+    Header[] headers = request.getHeaders(headerName);
+    if (headers.length > 0) {
+      StringBuilder result = new StringBuilder();
+      for (int i = 0; i < headers.length; i++) {
+        result.append(headers[i].getValue());
+        if (i + 1 < headers.length) {
+          result.append(",");
+        }
+      }
+      return result.toString();
     }
     return null;
   }
@@ -71,9 +78,16 @@ protected String getRequestHeader(HttpUriRequest request, String headerName) {
   protected String getResponseHeader(HttpContext context, String headerName) {
     final Object responseObject = context.getAttribute(HttpCoreContext.HTTP_RESPONSE);
     if (responseObject instanceof HttpResponse) {
-      Header header = ((HttpResponse) responseObject).getFirstHeader(headerName);
-      if (header != null) {
-        return header.getValue();
+      Header[] headers = ((HttpResponse) responseObject).getHeaders(headerName);
+      if (headers.length > 0) {
+        StringBuilder result = new StringBuilder();
+        for (int i = 0; i < headers.length; i++) {
+          result.append(headers[i].getValue());
+          if (i + 1 < headers.length) {
+            result.append(",");
+          }
+        }
+        return result.toString();
       }
     }
     return null;

dd-java-agent/instrumentation/apache-httpasyncclient-4/src/main/java/datadog/trace/instrumentation/apachehttpasyncclient/HostAndRequestAsHttpUriRequest.java

@@ -72,6 +72,11 @@ public Header getFirstHeader(String name) {
     return actualRequest.getFirstHeader(name);
   }
 
+  @Override
+  public Header[] getHeaders(String name) {
+    return actualRequest.getHeaders(name);
+  }
+
   public HttpRequest getActualRequest() {
     return actualRequest;
   }

dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientCallbackTest.groovy

@@ -1,4 +1,4 @@
-import datadog.trace.agent.test.base.HttpClientTest
+import datadog.trace.agent.test.base.HttpClientTest2
 import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions
 import datadog.trace.instrumentation.apachehttpasyncclient.ApacheHttpAsyncClientDecorator
 import org.apache.http.HttpResponse
@@ -14,7 +14,7 @@ import java.util.concurrent.CompletableFuture
 import java.util.concurrent.TimeUnit
 
 @Timeout(5)
-class ApacheHttpAsyncClientCallbackTest extends HttpClientTest implements TestingGenericHttpNamingConventions.ClientV0 {
+class ApacheHttpAsyncClientCallbackTest extends HttpClientTest2 implements TestingGenericHttpNamingConventions.ClientV0 {
 
   @Shared
   RequestConfig requestConfig = RequestConfig.custom()
@@ -31,11 +31,9 @@ class ApacheHttpAsyncClientCallbackTest extends HttpClientTest implements Testin
   }
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = new HttpUriRequest(method, uri)
-    headers.entrySet().each {
-      request.addHeader(new BasicHeader(it.key, it.value))
-    }
+    headers.each { request.addHeader(new BasicHeader(it[0], it[1])) }
 
     def responseFuture = new CompletableFuture<>()
 

dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientNullCallbackTest.groovy

@@ -1,4 +1,4 @@
-import datadog.trace.agent.test.base.HttpClientTest
+import datadog.trace.agent.test.base.HttpClientTest2
 import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions
 import datadog.trace.instrumentation.apachehttpasyncclient.ApacheHttpAsyncClientDecorator
 import org.apache.http.client.config.RequestConfig
@@ -11,7 +11,7 @@ import spock.lang.Timeout
 import java.util.concurrent.Future
 
 @Timeout(5)
-class ApacheHttpAsyncClientNullCallbackTest extends HttpClientTest implements TestingGenericHttpNamingConventions.ClientV0{
+class ApacheHttpAsyncClientNullCallbackTest extends HttpClientTest2 implements TestingGenericHttpNamingConventions.ClientV0{
 
   @Shared
   RequestConfig requestConfig = RequestConfig.custom()
@@ -28,11 +28,9 @@ class ApacheHttpAsyncClientNullCallbackTest extends HttpClientTest implements Te
   }
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = new HttpUriRequest(method, uri)
-    headers.entrySet().each {
-      request.addHeader(new BasicHeader(it.key, it.value))
-    }
+    headers.each { request.addHeader(new BasicHeader(it[0], it[1])) }
 
     // The point here is to test case when callback is null - fire-and-forget style
     // So to make sure request is done we start request, wait for future to finish

dd-java-agent/instrumentation/apache-httpasyncclient-4/src/test/groovy/ApacheHttpAsyncClientTest.groovy

@@ -1,5 +1,5 @@
 import datadog.trace.agent.test.asserts.TraceAssert
-import datadog.trace.agent.test.base.HttpClientTest
+import datadog.trace.agent.test.base.HttpClientTest2
 import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions
 import datadog.trace.instrumentation.apachehttpasyncclient.ApacheHttpAsyncClientDecorator
 import org.apache.http.HttpHost
@@ -16,7 +16,7 @@ import spock.lang.Timeout
 import java.util.concurrent.CountDownLatch
 
 @Timeout(5)
-abstract class ApacheHttpAsyncClientTest extends HttpClientTest {
+abstract class ApacheHttpAsyncClientTest extends HttpClientTest2 {
 
   @Shared
   RequestConfig requestConfig = RequestConfig.custom()
@@ -41,11 +41,9 @@ abstract class ApacheHttpAsyncClientTest extends HttpClientTest {
   }
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = createRequest(method, uri)
-    headers.entrySet().each {
-      request.addHeader(new BasicHeader(it.key, it.value))
-    }
+    headers.each { request.addHeader(new BasicHeader(it[0], it[1])) }
 
     def latch = new CountDownLatch(callback == null ? 0 : 1)
 

dd-java-agent/instrumentation/apache-httpclient-4/src/main/java/datadog/trace/instrumentation/apachehttpclient/ApacheHttpClientDecorator.java

@@ -46,18 +46,32 @@ protected int status(final HttpResponse httpResponse) {
 
   @Override
   protected String getRequestHeader(HttpUriRequest request, String headerName) {
-    Header header = request.getFirstHeader(headerName);
-    if (null != header) {
-      return header.getValue();
+    Header[] headers = request.getHeaders(headerName);
+    if (headers.length > 0) {
+      StringBuilder result = new StringBuilder();
+      for (int i = 0; i < headers.length; i++) {
+        result.append(headers[i].getValue());
+        if (i + 1 < headers.length) {
+          result.append(",");
+        }
+      }
+      return result.toString();
     }
     return null;
   }
 
   @Override
   protected String getResponseHeader(HttpResponse response, String headerName) {
-    Header header = response.getFirstHeader(headerName);
-    if (null != header) {
-      return header.getValue();
+    Header[] headers = response.getHeaders(headerName);
+    if (headers.length > 0) {
+      StringBuilder result = new StringBuilder();
+      for (int i = 0; i < headers.length; i++) {
+        result.append(headers[i].getValue());
+        if (i + 1 < headers.length) {
+          result.append(",");
+        }
+      }
+      return result.toString();
     }
     return null;
   }

dd-java-agent/instrumentation/apache-httpclient-4/src/main/java/datadog/trace/instrumentation/apachehttpclient/HostAndRequestAsHttpUriRequest.java

@@ -68,6 +68,11 @@ public Header getFirstHeader(String name) {
     return actualRequest.getFirstHeader(name);
   }
 
+  @Override
+  public Header[] getHeaders(String name) {
+    return actualRequest.getHeaders(name);
+  }
+
   public HttpRequest getActualRequest() {
     return actualRequest;
   }

dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientResponseHandlerTest.groovy

@@ -1,4 +1,4 @@
-import datadog.trace.agent.test.base.HttpClientTest
+import datadog.trace.agent.test.base.HttpClientTest2
 import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions
 import datadog.trace.instrumentation.apachehttpclient.ApacheHttpClientDecorator
 import org.apache.http.HttpResponse
@@ -11,7 +11,7 @@ import spock.lang.Shared
 import spock.lang.Timeout
 
 @Timeout(5)
-class ApacheHttpClientResponseHandlerTest extends HttpClientTest implements TestingGenericHttpNamingConventions.ClientV0 {
+class ApacheHttpClientResponseHandlerTest extends HttpClientTest2 implements TestingGenericHttpNamingConventions.ClientV0 {
 
   @Shared
   def client = new DefaultHttpClient()
@@ -31,11 +31,9 @@ class ApacheHttpClientResponseHandlerTest extends HttpClientTest implements Test
   }
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = new HttpUriRequest(method, uri)
-    headers.entrySet().each {
-      request.addHeader(new BasicHeader(it.key, it.value))
-    }
+    headers.each { request.addHeader(new BasicHeader(it[0], it[1])) }
 
     def status = client.execute(request, handler)
 

dd-java-agent/instrumentation/apache-httpclient-4/src/test/groovy/ApacheHttpClientTest.groovy

@@ -1,4 +1,4 @@
-import datadog.trace.agent.test.base.HttpClientTest
+import datadog.trace.agent.test.base.HttpClientTest2
 import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions
 import datadog.trace.instrumentation.apachehttpclient.ApacheHttpClientDecorator
 import org.apache.http.HttpHost
@@ -13,7 +13,7 @@ import org.apache.http.protocol.BasicHttpContext
 import spock.lang.Shared
 import spock.lang.Timeout
 
-abstract class ApacheHttpClientTest<T extends HttpRequest> extends HttpClientTest implements TestingGenericHttpNamingConventions.ClientV0 {
+abstract class ApacheHttpClientTest<T extends HttpRequest> extends HttpClientTest2 implements TestingGenericHttpNamingConventions.ClientV0 {
   @Shared
   def client = new DefaultHttpClient()
 
@@ -29,11 +29,9 @@ abstract class ApacheHttpClientTest<T extends HttpRequest> extends HttpClientTes
   }
 
   @Override
-  int doRequest(String method, URI uri, Map<String, String> headers, String body, Closure callback) {
+  int doRequest(String method, URI uri, List<List<String>> headers, String body, Closure callback) {
     def request = createRequest(method, uri)
-    headers.entrySet().each {
-      request.addHeader(new BasicHeader(it.key, it.value))
-    }
+    headers.each { request.addHeader(new BasicHeader(it[0], it[1])) }
 
     def response = executeRequest(request, uri)
     callback?.call()

dd-java-agent/instrumentation/apache-httpclient-5/src/main/java/datadog/trace/instrumentation/apachehttpclient5/ApacheHttpClientDecorator.java

@@ -4,6 +4,7 @@
 import datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 import org.apache.hc.core5.http.Header;
 import org.apache.hc.core5.http.HttpRequest;
 import org.apache.hc.core5.http.HttpResponse;
@@ -54,18 +55,37 @@ protected int status(final HttpResponse httpResponse) {
 
   @Override
   protected String getRequestHeader(HttpRequest request, String headerName) {
-    Header header = request.getFirstHeader(headerName);
-    if (null != header) {
-      return header.getValue();
+    System.out.println("headerName: " + headerName);
+    System.out.println("headers: " + Arrays.toString(request.getHeaders(headerName)));
+    Header[] headers = request.getHeaders(headerName);
+    if (headers.length > 0) {
+      StringBuilder result = new StringBuilder();
+      for (int i = 0; i < headers.length; i++) {
+        result.append(headers[i].getValue());
+        if (i + 1 < headers.length) {
+          result.append(",");
+        }
+      }
+      System.out.println("returning " + result);
+      return result.toString();
     }
+    System.out.println("returning null");
     return null;
   }
 
   @Override
   protected String getResponseHeader(HttpResponse response, String headerName) {
-    Header header = response.getFirstHeader(headerName);
-    if (null != header) {
-      return header.getValue();
+    Header[] headers = response.getHeaders(headerName);
+    if (headers.length > 0) {
+      StringBuilder result = new StringBuilder();
+      for (int i = 0; i < headers.length; i++) {
+        result.append(headers[i].getValue());
+        if (i + 1 < headers.length) {
+          result.append(",");
+        }
+      }
+      System.out.println("returning " + result);
+      return result.toString();
     }
     return null;
   }

dd-java-agent/instrumentation/apache-httpclient-5/src/main/java/datadog/trace/instrumentation/apachehttpclient5/HelperMethods.java

@@ -11,6 +11,7 @@
 import datadog.trace.bootstrap.instrumentation.api.AgentScope;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator;
+import java.util.Arrays;
 import org.apache.hc.client5.http.classic.HttpClient;
 import org.apache.hc.core5.http.HttpHost;
 import org.apache.hc.core5.http.HttpRequest;
@@ -32,13 +33,24 @@ public static AgentScope doMethodEnter(HttpHost host, HttpRequest request) {
       return null;
     }
 
+    System.out.println("----------doMethodEnter---------");
+    System.out.println("callDepth: " + callDepth);
+    System.out.println("host: " + host);
+    System.out.println("headers: " + Arrays.toString(request.getHeaders()));
+    System.out.println("-------------------");
+
     return activateHttpSpan(new HostAndRequestAsHttpUriRequest(host, request));
   }
 
   private static AgentScope activateHttpSpan(final HttpRequest request) {
     final AgentSpan span = startSpan(HTTP_REQUEST);
     final AgentScope scope = activateSpan(span);
 
+    System.out.println("----------activateHttpSpan---------");
+    System.out.println("request: " + request);
+    System.out.println("activateHttpSpan headers: " + Arrays.toString(request.getHeaders()));
+    System.out.println("-------------------");
+
     DECORATE.afterStart(span);
     DECORATE.onRequest(span, request);
 

dd-java-agent/instrumentation/apache-httpclient-5/src/main/java/datadog/trace/instrumentation/apachehttpclient5/HostAndRequestAsHttpUriRequest.java

@@ -15,6 +15,9 @@ public class HostAndRequestAsHttpUriRequest extends BasicClassicHttpRequest {
 
   public HostAndRequestAsHttpUriRequest(final HttpHost httpHost, final HttpRequest httpRequest) {
     super(httpRequest.getMethod(), httpHost, httpRequest.getPath());
+    System.out.println("----------HostAndRequestAsHttpUriRequest---------");
+    System.out.println("httpRequest: " + httpRequest);
+    System.out.println("-------------------");
     actualRequest = httpRequest;
     // Propagate in case the host or request is tainted
     PropagationUtils.taintObjectIfTainted(this, httpHost);
@@ -39,4 +42,9 @@ public void setHeader(String name, Object value) {
   public Header getFirstHeader(String name) {
     return actualRequest.getFirstHeader(name);
   }
+
+  @Override
+  public Header[] getHeaders(String name) {
+    return actualRequest.getHeaders(name);
+  }
 }