From d84f1c82f1b6a11dbdbe03520b1a38ec97c748ec Mon Sep 17 00:00:00 2001 From: Jay DeLuca Date: Sun, 20 Apr 2025 11:57:12 -0400 Subject: [PATCH 1/4] Add traces to testcontainer example --- examples/java/testcontainer/README.md | 32 ++++++---- examples/java/testcontainer/pom.xml | 2 +- .../grafana/example/TestcontainerTest.java | 61 ++++++++++++++++--- 3 files changed, 73 insertions(+), 22 deletions(-) diff --git a/examples/java/testcontainer/README.md b/examples/java/testcontainer/README.md index bf5882e9..0a6be221 100644 --- a/examples/java/testcontainer/README.md +++ b/examples/java/testcontainer/README.md @@ -1,33 +1,41 @@ # Using Testcontainers -The provided code demonstrates how to use **Testcontainers** with **Grafana's LGTM stack** to test OpenTelemetry metrics in a Java application. Here's a step-by-step explanation: +The provided code demonstrates how to use **Testcontainers** with **Grafana's LGTM stack** to test OpenTelemetry metrics +in a Java application. Here's a step-by-step explanation: 1. **Set Up the Testcontainers Environment**: - - The `@Testcontainers` annotation enables the Testcontainers extension for JUnit 5. - - The `@Container` annotation is used to define a `LgtmStackContainer` that runs the Grafana LGTM stack in a Docker container. +- The `@Testcontainers` annotation enables the Testcontainers extension for JUnit 5. +- The `@Container` annotation is used to define a `LgtmStackContainer` that runs the Grafana LGTM stack in a Docker + container. 2. **Configure OpenTelemetry**: - - In the `@BeforeEach` method, system properties are set to configure the OpenTelemetry exporter to send metrics to the LGTM stack running in the container. +- In the `@BeforeEach` method, system properties are set to configure the OpenTelemetry exporter to send metrics to + the LGTM stack running in the container. 3. **Run the Application**: - - The `OtelApp` class initializes OpenTelemetry and generates a custom metric (`sold_items`) with attributes (e.g., `tenant`). +- The `OtelApp` class initializes OpenTelemetry and generates a custom metric (`sold_items`) with attributes (e.g., + `tenant`) as well as a span representing the block the code. -4. **Test the Metrics Export**: +4. **Test Exporting Metrics and Traces**: - - The test method `testExportMetric` runs the application and queries the Prometheus endpoint in the LGTM stack to verify that the metric (`sold_items`) has been exported successfully. - - The `Awaitility` library is used to poll the Prometheus endpoint until the metric is found or a timeout occurs. +- The test method `testExportMetricsAndTraces` runs the application and queries the Prometheus endpoint and grafana + datasources in the LGTM stack to verify that the metric (`sold_items`) and span have been exported successfully. +- The `Awaitility` library is used to poll the endpoints until the telemetry is found or a timeout occurs. 5. **Debugging with Grafana**: - - The test outputs the Grafana URL (`lgtm.getGrafanaHttpUrl()`) to the console, allowing you to manually inspect the metrics in the Grafana UI. + +- The test outputs the Grafana URL (`lgtm.getGrafanaHttpUrl()`) to the console, allowing you to manually inspect the + telemetry in the Grafana UI if needed. ## Example Usage 1. Start the test using `mvn test`. 2. Check the console output for the Grafana URL. -3. Open the Grafana UI, navigate to the Explore tab, and query the metrics. -4. The test will pass if the metric is successfully exported and found in Prometheus. +3. Open the Grafana UI, navigate to the Explore tab, and query the metrics or traces. +4. The test will pass if the metric and span are successfully exported and found in Prometheus and Tempo. -This setup is useful for validating OpenTelemetry instrumentation and ensuring metrics are correctly exported to a monitoring system. +This setup is useful for validating OpenTelemetry instrumentation and ensuring metrics are correctly exported to a +monitoring system. \ No newline at end of file diff --git a/examples/java/testcontainer/pom.xml b/examples/java/testcontainer/pom.xml index bec36342..3c714144 100644 --- a/examples/java/testcontainer/pom.xml +++ b/examples/java/testcontainer/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.grafana.example - testcontaier + testcontainer 1.0.0-SNAPSHOT Java Testcontainer Demo diff --git a/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java b/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java index 27758baf..2a1539bf 100644 --- a/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java +++ b/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java @@ -2,6 +2,7 @@ import static org.awaitility.Awaitility.await; +import java.io.IOException; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; @@ -9,11 +10,15 @@ import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Base64; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.grafana.LgtmStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; @Testcontainers public class TestcontainerTest { @@ -29,10 +34,10 @@ void setUp() { } @Test - void testExportMetric() { - // Howto: - // 1. The test with a really long timeout - // 2. Go to the Grafana UI + void testExportMetricsAndTraces() throws IOException, InterruptedException { + // How to debug: + // 1. Run the test with a really long timeout (update the awaitility argument) + // 2. Go to the Grafana UI (login with admin:admin) // 3. Open the Explore tab // 4. Select the Prometheus data source // 5. Find your metric by name or attribute (e.g. "tenant1") @@ -43,19 +48,17 @@ void testExportMetric() { var app = new OtelApp(); app.run(); + HttpClient client = HttpClient.newHttpClient(); String query = URLEncoder.encode( - "sold_items_total{job=\"otel-java-test\",service_name=\"otel-java-test\",tenant=\"tenant1\"}", + "sold_items_total{job=\"otel-java-test\",tenant=\"tenant1\"}", StandardCharsets.UTF_8); String prometheusHttpUrl = lgtm.getPromehteusHttpUrl() + "/api/v1/query?query=" + query; - HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder().uri(URI.create(prometheusHttpUrl)).build(); - // Total time: 18.448 s (for the whole test, will take longer when the image needs to be - // downloaded) await() - .atMost(Duration.ofSeconds(10)) + .atMost(Duration.ofSeconds(20)) .until( () -> { HttpResponse response = @@ -63,5 +66,45 @@ void testExportMetric() { String body = response.body(); return response.statusCode() == 200 && body.contains("sold_items"); }); + + // Get the Tempo datasource ID so we can query traces from grafana + String authHeader = "Basic " + Base64.getEncoder().encodeToString("admin:admin".getBytes(StandardCharsets.UTF_8)); + HttpRequest dsRequest = HttpRequest.newBuilder() + .uri(URI.create(lgtm.getGrafanaHttpUrl() + "/api/datasources")) + .header("Authorization", authHeader) + .build(); + + HttpResponse dsResponse = client.send(dsRequest, HttpResponse.BodyHandlers.ofString()); + String dsResponseBody = dsResponse.body(); + + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode rootNode = objectMapper.readTree(dsResponseBody); + + assert rootNode != null; + + // Parse the JSON response to extract the Tempo datasource ID + Integer tempoDsId = null; + for (JsonNode node : rootNode) { + if (node.get("type").asText().equals("tempo")) { + tempoDsId = node.get("id").asInt(); + } + } + assert tempoDsId != null : "Tempo datasource ID not found in the response"; + + // Query for traces + String queryUrl = lgtm.getGrafanaHttpUrl() + "/api/datasources/proxy/" + tempoDsId + "/api/search?tags=service.name=otel-java-test"; + HttpRequest queryRequest = HttpRequest.newBuilder() + .uri(URI.create(queryUrl)) + .header("Authorization", authHeader) + .build(); + + await() + .atMost(Duration.ofSeconds(10)) + .until( + () -> { + HttpResponse response = client.send(queryRequest, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + return response.statusCode() == 200 && body.contains("otel-java-test"); + }); } } From 97d9eee247b4edfb972cbb531e573abee70bcab9 Mon Sep 17 00:00:00 2001 From: Jay DeLuca Date: Fri, 25 Apr 2025 05:54:27 -0400 Subject: [PATCH 2/4] hit tempo directly --- examples/java/testcontainer/pom.xml | 2 +- .../grafana/example/TestcontainerTest.java | 49 +++---------------- 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/examples/java/testcontainer/pom.xml b/examples/java/testcontainer/pom.xml index 3c714144..f9427624 100644 --- a/examples/java/testcontainer/pom.xml +++ b/examples/java/testcontainer/pom.xml @@ -27,7 +27,7 @@ org.testcontainers testcontainers-bom - 1.20.6 + 1.21.0 pom import diff --git a/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java b/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java index 2a1539bf..70662bea 100644 --- a/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java +++ b/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java @@ -2,7 +2,6 @@ import static org.awaitility.Awaitility.await; -import java.io.IOException; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; @@ -10,15 +9,12 @@ import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.Base64; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.grafana.LgtmStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; @Testcontainers public class TestcontainerTest { @@ -31,34 +27,34 @@ void setUp() { System.setProperty("otel.exporter.otlp.protocol", "http/protobuf"); System.setProperty("otel.resource.attributes", "service.name=otel-java-test"); System.setProperty("otel.metric.export.interval", "1s"); + System.setProperty("otel.bsp.schedule.delay", "500ms"); } @Test - void testExportMetricsAndTraces() throws IOException, InterruptedException { + void testExportMetricsAndTraces() throws InterruptedException { // How to debug: // 1. Run the test with a really long timeout (update the awaitility argument) - // 2. Go to the Grafana UI (login with admin:admin) + // 2. Go to the Grafana UI // 3. Open the Explore tab // 4. Select the Prometheus data source // 5. Find your metric by name or attribute (e.g. "tenant1") // 6. Click on the metric to see the details // 7. Copy the query and paste it into the test System.out.println("Grafana URL to debug telemetry: " + lgtm.getGrafanaHttpUrl()); - var app = new OtelApp(); app.run(); HttpClient client = HttpClient.newHttpClient(); String query = URLEncoder.encode( - "sold_items_total{job=\"otel-java-test\",tenant=\"tenant1\"}", + "sold_items_total{job=\"otel-java-test\",service_name=\"otel-java-test\",tenant=\"tenant1\"}", StandardCharsets.UTF_8); - String prometheusHttpUrl = lgtm.getPromehteusHttpUrl() + "/api/v1/query?query=" + query; + String prometheusHttpUrl = lgtm.getPrometheusHttpUrl() + "/api/v1/query?query=" + query; HttpRequest request = HttpRequest.newBuilder().uri(URI.create(prometheusHttpUrl)).build(); await() - .atMost(Duration.ofSeconds(20)) + .atMost(Duration.ofSeconds(10)) .until( () -> { HttpResponse response = @@ -67,42 +63,13 @@ void testExportMetricsAndTraces() throws IOException, InterruptedException { return response.statusCode() == 200 && body.contains("sold_items"); }); - // Get the Tempo datasource ID so we can query traces from grafana - String authHeader = "Basic " + Base64.getEncoder().encodeToString("admin:admin".getBytes(StandardCharsets.UTF_8)); - HttpRequest dsRequest = HttpRequest.newBuilder() - .uri(URI.create(lgtm.getGrafanaHttpUrl() + "/api/datasources")) - .header("Authorization", authHeader) - .build(); - - HttpResponse dsResponse = client.send(dsRequest, HttpResponse.BodyHandlers.ofString()); - String dsResponseBody = dsResponse.body(); - - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode rootNode = objectMapper.readTree(dsResponseBody); - - assert rootNode != null; - - // Parse the JSON response to extract the Tempo datasource ID - Integer tempoDsId = null; - for (JsonNode node : rootNode) { - if (node.get("type").asText().equals("tempo")) { - tempoDsId = node.get("id").asInt(); - } - } - assert tempoDsId != null : "Tempo datasource ID not found in the response"; - - // Query for traces - String queryUrl = lgtm.getGrafanaHttpUrl() + "/api/datasources/proxy/" + tempoDsId + "/api/search?tags=service.name=otel-java-test"; - HttpRequest queryRequest = HttpRequest.newBuilder() - .uri(URI.create(queryUrl)) - .header("Authorization", authHeader) - .build(); + HttpRequest traceRequest = HttpRequest.newBuilder().uri(URI.create(String.format("%s/api/search", lgtm.getTempoUrl()))).build(); await() .atMost(Duration.ofSeconds(10)) .until( () -> { - HttpResponse response = client.send(queryRequest, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = client.send(traceRequest, HttpResponse.BodyHandlers.ofString()); String body = response.body(); return response.statusCode() == 200 && body.contains("otel-java-test"); }); From 5c8344a657d38933117f769bcb963b6e4f0b915a Mon Sep 17 00:00:00 2001 From: Jay DeLuca Date: Fri, 25 Apr 2025 05:55:34 -0400 Subject: [PATCH 3/4] readme fix --- examples/java/testcontainer/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/java/testcontainer/README.md b/examples/java/testcontainer/README.md index 0a6be221..a5256e7f 100644 --- a/examples/java/testcontainer/README.md +++ b/examples/java/testcontainer/README.md @@ -21,8 +21,8 @@ in a Java application. Here's a step-by-step explanation: 4. **Test Exporting Metrics and Traces**: -- The test method `testExportMetricsAndTraces` runs the application and queries the Prometheus endpoint and grafana - datasources in the LGTM stack to verify that the metric (`sold_items`) and span have been exported successfully. +- The test method `testExportMetricsAndTraces` runs the application and queries the Prometheus and Tempo endpoints + in the LGTM stack to verify that the metric (`sold_items`) and span have been exported successfully. - The `Awaitility` library is used to poll the endpoints until the telemetry is found or a timeout occurs. 5. **Debugging with Grafana**: From 7a50a8c08e361441cdb68dd865f424b2647493fb Mon Sep 17 00:00:00 2001 From: Jay DeLuca Date: Tue, 29 Apr 2025 21:03:30 -0400 Subject: [PATCH 4/4] reformat --- .../test/java/com/grafana/example/TestcontainerTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java b/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java index 70662bea..a84b3c5d 100644 --- a/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java +++ b/examples/java/testcontainer/src/test/java/com/grafana/example/TestcontainerTest.java @@ -19,7 +19,8 @@ @Testcontainers public class TestcontainerTest { - @Container private final LgtmStackContainer lgtm = new LgtmStackContainer("grafana/otel-lgtm"); + @Container + private final LgtmStackContainer lgtm = new LgtmStackContainer("grafana/otel-lgtm"); @BeforeEach void setUp() { @@ -63,7 +64,9 @@ void testExportMetricsAndTraces() throws InterruptedException { return response.statusCode() == 200 && body.contains("sold_items"); }); - HttpRequest traceRequest = HttpRequest.newBuilder().uri(URI.create(String.format("%s/api/search", lgtm.getTempoUrl()))).build(); + HttpRequest traceRequest = HttpRequest.newBuilder() + .uri(URI.create(String.format("%s/api/search", lgtm.getTempoUrl()))) + .build(); await() .atMost(Duration.ofSeconds(10))