diff --git a/bom/application/pom.xml b/bom/application/pom.xml index 9449c48011b9c..17e37e6a8909e 100644 --- a/bom/application/pom.xml +++ b/bom/application/pom.xml @@ -226,6 +226,8 @@ 1.1.0 3.0.0 2.12.3 + + 0.16.0 1.0.10 @@ -490,6 +492,18 @@ import + + + io.prometheus + simpleclient + ${prometheus.version} + + + io.prometheus + simpleclient_common + ${prometheus.version} + + @@ -2991,6 +3005,42 @@ quarkus-virtual-threads-deployment ${project.version} + + io.quarkus + quarkus-observability-devservices-common + ${project.version} + + + io.quarkus + quarkus-observability-devservices + ${project.version} + + + io.quarkus + quarkus-observability-devservices-lgtm + ${project.version} + provided + + + io.quarkus + quarkus-observability-testcontainers + ${project.version} + + + io.quarkus + quarkus-observability-devresource-common + ${project.version} + + + io.quarkus + quarkus-observability-devresource-testcontainers + ${project.version} + + + io.quarkus + quarkus-observability-devresource-lgtm + ${project.version} + diff --git a/core/deployment/src/main/java/io/quarkus/deployment/Feature.java b/core/deployment/src/main/java/io/quarkus/deployment/Feature.java index 14b9fd7c89bf8..25c5149188eb4 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/Feature.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/Feature.java @@ -69,6 +69,7 @@ public enum Feature { NARAYANA_LRA, NARAYANA_STM, NEO4J, + OBSERVABILITY, OIDC, OIDC_CLIENT, RESTEASY_CLIENT_OIDC_FILTER, diff --git a/core/deployment/src/main/java/io/quarkus/deployment/console/StartupLogCompressor.java b/core/deployment/src/main/java/io/quarkus/deployment/console/StartupLogCompressor.java index 7576025324987..afd9f86a17fc9 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/console/StartupLogCompressor.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/console/StartupLogCompressor.java @@ -3,6 +3,7 @@ import java.io.Closeable; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; @@ -26,18 +27,28 @@ public class StartupLogCompressor implements Closeable, BiPredicate toDump = new ArrayList<>(); final AtomicInteger COUNTER = new AtomicInteger(); final Predicate additionalThreadPredicate; + final Predicate linePredicate; // test if we always print the line / log public StartupLogCompressor(String name, @SuppressWarnings("unused") Optional consoleInstalledBuildItem, @SuppressWarnings("unused") LoggingSetupBuildItem loggingSetupBuildItem) { - this(name, consoleInstalledBuildItem, loggingSetupBuildItem, (s) -> false); + this(name, consoleInstalledBuildItem, loggingSetupBuildItem, s -> false); } public StartupLogCompressor(String name, @SuppressWarnings("unused") Optional consoleInstalledBuildItem, @SuppressWarnings("unused") LoggingSetupBuildItem loggingSetupBuildItem, Predicate additionalThreadPredicate) { - this.additionalThreadPredicate = additionalThreadPredicate; + this(name, consoleInstalledBuildItem, loggingSetupBuildItem, additionalThreadPredicate, s -> false); + } + + public StartupLogCompressor(String name, + @SuppressWarnings("unused") Optional consoleInstalledBuildItem, + @SuppressWarnings("unused") LoggingSetupBuildItem loggingSetupBuildItem, + Predicate additionalThreadPredicate, + Predicate linePredicate) { + this.additionalThreadPredicate = Objects.requireNonNull(additionalThreadPredicate); + this.linePredicate = Objects.requireNonNull(linePredicate); if (QuarkusConsole.INSTANCE.isAnsiSupported()) { QuarkusConsole.installRedirects(); this.name = name; @@ -74,8 +85,8 @@ public void closeAndDumpCaptured() { @Override public boolean test(String s, Boolean errorStream) { - if (thread == null) { - //not installed + if (thread == null || linePredicate.test(s)) { + //not installed or line predicate tested to true return true; } Thread current = Thread.currentThread(); diff --git a/core/runtime/src/main/java/io/quarkus/runtime/util/EnumerationUtil.java b/core/runtime/src/main/java/io/quarkus/runtime/util/EnumerationUtil.java new file mode 100644 index 0000000000000..6dbab086bb060 --- /dev/null +++ b/core/runtime/src/main/java/io/quarkus/runtime/util/EnumerationUtil.java @@ -0,0 +1,69 @@ +package io.quarkus.runtime.util; + +import java.util.Enumeration; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Spliterator; +import java.util.function.Consumer; +import java.util.stream.Stream; + +/** + * Transform to "old school" Enumeration from Iterator/Spliterator/Stream + */ +public class EnumerationUtil { + public static Enumeration from(Iterator iterator) { + Objects.requireNonNull(iterator); + + return new Enumeration() { + @Override + public boolean hasMoreElements() { + return iterator.hasNext(); + } + + @Override + public T nextElement() { + return iterator.next(); + } + }; + } + + public static Enumeration from(Spliterator spliterator) { + Objects.requireNonNull(spliterator); + + class Adapter implements Enumeration, Consumer { + boolean valueReady; + T nextElement; + + public void accept(T t) { + this.valueReady = true; + this.nextElement = t; + } + + public boolean hasMoreElements() { + if (!this.valueReady) { + spliterator.tryAdvance(this); + } + + return this.valueReady; + } + + public T nextElement() { + if (!this.valueReady && !this.hasMoreElements()) { + throw new NoSuchElementException(); + } else { + this.valueReady = false; + T t = this.nextElement; + this.nextElement = null; + return t; + } + } + } + + return new Adapter(); + } + + public static Enumeration from(Stream stream) { + return from(stream.spliterator()); + } +} diff --git a/devtools/bom-descriptor-json/pom.xml b/devtools/bom-descriptor-json/pom.xml index 426891e36c7da..b86dea58cf920 100644 --- a/devtools/bom-descriptor-json/pom.xml +++ b/devtools/bom-descriptor-json/pom.xml @@ -1591,6 +1591,19 @@ + + io.quarkus + quarkus-observability-devservices + ${project.version} + pom + test + + + * + * + + + io.quarkus quarkus-oidc diff --git a/docs/pom.xml b/docs/pom.xml index e70a317eb71e5..d5d80164248f1 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -1607,6 +1607,19 @@ + + io.quarkus + quarkus-observability-devservices-deployment + ${project.version} + pom + test + + + * + * + + + io.quarkus quarkus-oidc-deployment diff --git a/docs/src/main/asciidoc/dev-services.adoc b/docs/src/main/asciidoc/dev-services.adoc index 01fd6fdf4e77b..77014c01a1b61 100644 --- a/docs/src/main/asciidoc/dev-services.adoc +++ b/docs/src/main/asciidoc/dev-services.adoc @@ -165,6 +165,14 @@ More information can be found in the xref:elasticsearch-dev-services.adoc[Elasti include::{generated-dir}/config/quarkus-elasticsearch-devservices-elasticsearch-dev-services-build-time-config.adoc[opts=optional, leveloffset=+1] +=== Observability + +The Observability Dev Services will be enabled when the `quarkus-observability-devservices` extension is present in your application, and +there is at least one dev resource on the classpath. More information can be found in the +xref:observability-devservices.adoc[Observability Dev Services Guide]. + +include::{generated-dir}/config/quarkus-observability-config-observability-configuration.adoc[opts=optional, leveloffset=+1] + == Dev Services beyond the Quarkus Platform Many Quarkiverse extensions which are not in the Quarkus Platform also offer Dev Services. diff --git a/docs/src/main/asciidoc/observability-devservices-lgtm.adoc b/docs/src/main/asciidoc/observability-devservices-lgtm.adoc new file mode 100644 index 0000000000000..df199401954e4 --- /dev/null +++ b/docs/src/main/asciidoc/observability-devservices-lgtm.adoc @@ -0,0 +1,282 @@ +//// +This guide is maintained in the main Quarkus repository +and pull requests should be submitted there: +https://github.com/quarkusio/quarkus/tree/main/docs/src/main/asciidoc +//// += Observability Dev Services with Grafana OTel LGTM + +include::_attributes.adoc[] +:categories: observability,devservices,telemetry,metrics,tracing,logging, opentelemetry, micrometer, prometheus, tempo, loki, grafana +:summary: Instructions on how to use Grafana Otel LGTM +:topics: observability,grafana,lgtm,otlp,opentelemetry,devservices,micrometer +:extensions: io.quarkus:quarkus-observability-devservices + +https://github.com/grafana/docker-otel-lgtm[OTel-LGTM] is `all-in-one` Docker image containing OpenTelemetry's https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/README.md[OTLP] as the protocol to transport metrics, tracing and logging data to an https://opentelemetry.io/docs/collector[OpenTelemetry Collector] which then stores signals data into https://prometheus.io/[Prometheus] (metrics), https://github.com/grafana/tempo[Tempo] (traces) and https://github.com/grafana/loki[Loki] (logs), only to have it visualized by https://github.com/grafana/grafana[Grafana]. It's used by Quarkus Observability to provide the Grafana OTel LGTM Dev Resource. + +== Configuring your project + +Add the Quarkus Grafana OTel LGTM sink (where data goes) extension to your build file: + +[source,xml,role="primary asciidoc-tabs-target-sync-cli asciidoc-tabs-target-sync-maven"] +.pom.xml +---- + + io.quarkus + quarkus-observability-devservices-lgtm + provided + +---- + +[source,gradle,role="secondary asciidoc-tabs-target-sync-gradle"] +.build.gradle +---- +implementation("quarkus-observability-devservices-lgtm") +---- + +=== Metrics + +If you're using https://micrometer.io/[MicroMeter's] Quarkiverse OTLP registry to push metrics to Grafana OTel LGTM, this is how you would define the export endpoint url; where `quarkus.otel-collector.url` is provided by the Observability Dev Services extension. + +[source,properties] +---- +# Micrometer OTLP registry +%test.quarkus.micrometer.export.otlp.url=http://${quarkus.otel-collector.url}/v1/metrics +%dev.quarkus.micrometer.export.otlp.url=http://${quarkus.otel-collector.url}/v1/metrics +%prod.quarkus.micrometer.export.otlp.url=http://localhost:4318/v1/metrics +---- +Please note that the `${quarkus.otel-collector.url}` value is generated by quarkus when it starts the Grafana OTel LGTM Dev Resource. + +Along OTel collector enpoint url, LGTM Dev Resource also provides a Grafana endpoint url - under `quarkus.grafana.url` property. + +In this case LGTM Dev Resource would be automatically started and used by Observability Dev Services. + +If you don't want all the hassle with Dev Services (e.g. lookup and re-use of existing running containers, etc) you can simply disable Dev Services and enable just Dev Resource usage: + +[source,properties] +---- +quarkus.observability.enabled=false +quarkus.observability.dev-resources=true +---- + +=== Tracing + +Just add the quarkus-opentelemetry extension to your build file: +[source,xml,role="primary asciidoc-tabs-target-sync-cli asciidoc-tabs-target-sync-maven"] +.pom.xml +---- + + io.quarkus + quarkus-opentelemetry + +---- + +[source,gradle,role="secondary asciidoc-tabs-target-sync-gradle"] +.build.gradle +---- +implementation("io.quarkus:quarkus-opentelemetry") +---- + +On the `application.properties` file, you can define: +[source,properties] +---- +# OpenTelemetry +quarkus.otel.exporter.otlp.traces.protocol=http/protobuf +%test.quarkus.otel.exporter.otlp.traces.endpoint=http://${quarkus.otel-collector.url} +%dev.quarkus.otel.exporter.otlp.traces.endpoint=http://${quarkus.otel-collector.url} +%prod.quarkus.otel.exporter.otlp.traces.endpoint=http://localhost:4318 +---- +=== Access Grafana + +Once you start your app in dev mode: + +include::{includes}/devtools/dev.adoc[] + +You will see a message like this: + +[source, log] +---- +Lgtm Dev Services Starting: 2024-02-20 11:15:24,540 INFO [org.tes.con.wai.str.HttpWaitStrategy] (build-32) /loving_chatelet: Waiting for 60 seconds for URL: http://localhost:61907/ (where port 61907 maps to container port 3000) +---- +Remember that Grafana is accessible in an ephemeral port, so you need to check the logs to see which port is being used. In this example, it's port 61907. + +If you miss the message you can always check the port with this Docker command: +[source, bash] +---- +docker ps | grep grafana +---- +=== Tests + +And for the least 'auto-magical' usage in the tests, simply disable both (Dev Resources are already disabled by default): + +[source,properties] +---- +quarkus.observability.enabled=false +---- + +And then explicitly list LGTM Dev Resource in the test as a `@QuarkusTestResource` resource: +[source, java] +---- +@QuarkusTest +@QuarkusTestResource(value = LgtmResource.class, restrictToAnnotatedClass = true) +@TestProfile(QuarkusTestResourceTestProfile.class) +public class LgtmLifecycleTest extends LgtmTestBase { +} +---- + +== Testing full Grafana OTel LGTM stack - example + +Use existing Quarkus MicroMeter OTLP registry + +[source,xml,role="primary asciidoc-tabs-target-sync-cli asciidoc-tabs-target-sync-maven"] +.pom.xml +---- + + io.quarkiverse.micrometer.registry + quarkus-micrometer-registry-otlp + +---- + +[source,gradle,role="secondary asciidoc-tabs-target-sync-gradle"] +.build.gradle +---- +implementation("io.quarkiverse.micrometer.registry:quarkus-micrometer-registry-otlp") +---- + +On the test `application.properties` file, you need to define: +[source,properties] +---- +# Micrometer OTLP registry +quarkus.micrometer.export.otlp.url=http://${quarkus.otel-collector.url}/v1/metrics +# OpenTelemetry +quarkus.otel.exporter.otlp.traces.protocol=http/protobuf +quarkus.otel.exporter.otlp.traces.endpoint=http://${quarkus.otel-collector.url} +---- + +Simply inject the Meter registry into your code -- it will periodically push metrics to Grafana LGTM's OTLP HTTP endpoint. + +[source, java] +---- +@Path("/api") +public class SimpleEndpoint { + private static final Logger log = Logger.getLogger(SimpleEndpoint.class); + + @Inject + MeterRegistry registry; + + @PostConstruct + public void start() { + Gauge.builder("xvalue", arr, a -> arr[0]) + .baseUnit("X") + .description("Some random x") + .tag("my_key", "x") + .register(registry); + } + + // ... +} +---- + +Where you can then check Grafana's datasource API for existing metrics data. + +[source, java] +---- +public class LgtmTestBase { + + @ConfigProperty(name = "quarkus.grafana.url") + String url; // NOTE -- injected Grafana endpoint url! + + @Test + public void testTracing() { + String response = RestAssured.get("/api/poke?f=100").body().asString(); + System.out.println(response); + GrafanaClient client = new GrafanaClient("http://" + url, "admin", "admin"); + Awaitility.await().atMost(61, TimeUnit.SECONDS).until( + client::user, + u -> "admin".equals(u.login)); + Awaitility.await().atMost(61, TimeUnit.SECONDS).until( + () -> client.query("xvalue_X"), + result -> !result.data.result.isEmpty()); + } + +} + +// simple Grafana HTTP client + +public class GrafanaClient { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final String url; + private final String username; + private final String password; + + public GrafanaClient(String url, String username, String password) { + this.url = url; + this.username = username; + this.password = password; + } + + private void handle( + String path, + Function method, + HttpResponse.BodyHandler bodyHandler, + BiConsumer, T> consumer) { + try { + String credentials = username + ":" + password; + String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes()); + + HttpClient httpClient = HttpClient.newHttpClient(); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url + path)) + .header("Authorization", "Basic " + encodedCredentials); + HttpRequest request = method.apply(builder).build(); + + HttpResponse response = httpClient.send(request, bodyHandler); + int code = response.statusCode(); + if (code < 200 || code > 299) { + throw new IllegalStateException("Bad response: " + code + " >> " + response.body()); + } + consumer.accept(response, response.body()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + public User user() { + AtomicReference ref = new AtomicReference<>(); + handle( + "/api/user", + HttpRequest.Builder::GET, + HttpResponse.BodyHandlers.ofString(), + (r, b) -> { + try { + User user = MAPPER.readValue(b, User.class); + ref.set(user); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + }); + return ref.get(); + } + + public QueryResult query(String query) { + AtomicReference ref = new AtomicReference<>(); + handle( + "/api/datasources/proxy/1/api/v1/query?query=" + query, + HttpRequest.Builder::GET, + HttpResponse.BodyHandlers.ofString(), + (r, b) -> { + try { + QueryResult result = MAPPER.readValue(b, QueryResult.class); + ref.set(result); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + }); + return ref.get(); + } +} + +---- diff --git a/docs/src/main/asciidoc/observability-devservices.adoc b/docs/src/main/asciidoc/observability-devservices.adoc new file mode 100644 index 0000000000000..e906b22741cd6 --- /dev/null +++ b/docs/src/main/asciidoc/observability-devservices.adoc @@ -0,0 +1,42 @@ +//// +This guide is maintained in the main Quarkus repository +and pull requests should be submitted there: +https://github.com/quarkusio/quarkus/tree/main/docs/src/main/asciidoc +//// += Observability Dev Services +include::_attributes.adoc[] +:categories: observability,devservices,telemetry,metrics,tracing,logging +:summary: Entry point for Observability DevServices +:topics: observability,grafana,lgtm,prometheus,victoriametrics,jaeger,otel,otlp +:extensions: io.quarkus:quarkus-observability-devservices + +We are already familiar with xref:dev-services.adoc[Dev Service] concept, but in the case of Observability we need a way to orchestrate and connect more than a single dev service, usually a whole stack of them; e.g. a metrics agent periodically scraping application for metrics, pushing them into timeseries database, and Grafana feeding graphs of this timeseries data. + +With this in mind, we added a new concept of Dev Resource, an adapter between Dev Service concept and https://testcontainers.com/[Testcontainers]. And since we now have fine-grained services - with the Dev Resource per container, we can take this even further, allowing the user to choose the way to use this new Dev Resource concept: + +NOTE: Each Dev Resource implementation is an `@QuarkusTestResourceLifecycleManager` implementation as well + +* leave it to Dev Services to pick-up various Dev Resources from classpath, and apply xref:dev-services.adoc[Dev Service] concept to it + +* explicitly disable Dev Services and enable Dev Resources and use less-heavy concept of starting and stopping Dev Resources + +* explicitly disable both Dev Services and Dev Resources, and use Quarkus' `@QuarkusTestResource` testing concept (see Note) + +You can either add Observability extension dependency along with needed Dev Resources dependencies, or you use existing `sinks` - pom.xml files which add Observability extension dependency along with other required dependencies for certain technology stacks; e.g. `victoriametrics` sink would have `quarkus-observability-devresource-victoriametrics` and `quarkus-victoriametrics-client` dependencies already included in the `pom.xml`. + +[NOTE] +==== +Make sure you set the `scope` of these sink dependencies to `provided`, otherwise libraries such as Testcontainers will end-up in your app's production libraries: +[source, xml] +---- + + io.quarkus + quarkus-observability-devservices-... + provided + +---- +==== + +Let's see how all of this looks in practice, with the usual `all-in-one` Grafana usage, in the form of https://github.com/grafana/docker-otel-lgtm[OTel-LGTM] Docker image. + +* xref:observability-devservices-lgtm.adoc[Getting Started with Grafana-OTel-LGTM] diff --git a/docs/src/main/asciidoc/opentelemetry.adoc b/docs/src/main/asciidoc/opentelemetry.adoc index 8616bad99244c..5462114043b5b 100644 --- a/docs/src/main/asciidoc/opentelemetry.adoc +++ b/docs/src/main/asciidoc/opentelemetry.adoc @@ -167,7 +167,21 @@ If you need to enable or disable the exporter at runtime, you can use the < locateContainer(String serviceName, boolean sh } } + /** + * @return container id, if exists + */ + public Optional locateContainer(String serviceName, boolean shared, LaunchMode launchMode, + BiConsumer consumer) { + if (shared && launchMode == LaunchMode.DEVELOPMENT) { + return lookup(serviceName) + .map(container -> { + Arrays.stream(container.getPorts()) + .filter(cp -> Objects.nonNull(cp.getPublicPort()) && Objects.nonNull(cp.getPrivatePort())) + .forEach(cp -> { + ContainerAddress containerAddress = new ContainerAddress( + container.getId(), + DockerClientFactory.instance().dockerHostIpAddress(), + cp.getPublicPort()); + consumer.accept(cp.getPrivatePort(), containerAddress); + }); + return container.getId(); + }); + } else { + return Optional.empty(); + } + } + public Optional locatePublicPort(String serviceName, boolean shared, LaunchMode launchMode, int privatePort) { if (shared && launchMode == LaunchMode.DEVELOPMENT) { return lookup(serviceName) diff --git a/extensions/observability-devservices/common/pom.xml b/extensions/observability-devservices/common/pom.xml new file mode 100644 index 0000000000000..0ad62360c8ce4 --- /dev/null +++ b/extensions/observability-devservices/common/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + quarkus-observability-devservices-parent + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-devservices-common + Quarkus - Observability Dev Services - Common + + + + io.quarkus + quarkus-core + provided + + + io.smallrye.config + smallrye-config-core + provided + + + diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/ContainerConstants.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/ContainerConstants.java new file mode 100644 index 0000000000000..630f29824e42a --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/ContainerConstants.java @@ -0,0 +1,15 @@ +package io.quarkus.observability.common; + +public final class ContainerConstants { + + // Images + + public static final String LGTM = "docker.io/grafana/otel-lgtm:0.4.0"; + + // Ports + + public static final int GRAFANA_PORT = 3000; + + public static final int OTEL_GRPC_EXPORTER_PORT = 4317; + public static final int OTEL_HTTP_EXPORTER_PORT = 4318; +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/AbstractContainerConfig.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/AbstractContainerConfig.java new file mode 100644 index 0000000000000..3ddefb2292994 --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/AbstractContainerConfig.java @@ -0,0 +1,51 @@ +package io.quarkus.observability.common.config; + +import java.util.Locale; +import java.util.Optional; +import java.util.Set; + +public abstract class AbstractContainerConfig implements ContainerConfig { + + private final String imageName; + private final boolean shared; + + public AbstractContainerConfig(String imageName) { + this(imageName, true); + } + + public AbstractContainerConfig(String imageName, boolean shared) { + this.imageName = imageName; + this.shared = shared; + } + + @Override + public boolean enabled() { + return true; + } + + @Override + public String imageName() { + return imageName; + } + + @Override + public boolean shared() { + return shared; + } + + @Override + public Optional> networkAliases() { + return Optional.empty(); + } + + @Override + public String label() { + String sn = getClass().getSimpleName().toLowerCase(Locale.ROOT); + return "quarkus-dev-resource-" + sn; + } + + @Override + public String serviceName() { + return "quarkus"; + } +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/AbstractGrafanaConfig.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/AbstractGrafanaConfig.java new file mode 100644 index 0000000000000..b73cea0232961 --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/AbstractGrafanaConfig.java @@ -0,0 +1,51 @@ +package io.quarkus.observability.common.config; + +import java.time.Duration; + +import io.quarkus.observability.common.ContainerConstants; + +public abstract class AbstractGrafanaConfig extends AbstractContainerConfig implements GrafanaConfig { + + private final String username; + private final String password; + private final int grafanaPort; + + public AbstractGrafanaConfig(String imageName) { + this(imageName, true, "admin", "admin", ContainerConstants.GRAFANA_PORT); + } + + public AbstractGrafanaConfig(String imageName, boolean shared) { + this(imageName, shared, "admin", "admin", ContainerConstants.GRAFANA_PORT); + } + + public AbstractGrafanaConfig(String imageName, String username, String password, int grafanaPort) { + this(imageName, true, username, password, grafanaPort); + } + + public AbstractGrafanaConfig(String imageName, boolean shared, String username, String password, int grafanaPort) { + super(imageName, shared); + this.username = username; + this.password = password; + this.grafanaPort = grafanaPort; + } + + @Override + public String username() { + return username; + } + + @Override + public String password() { + return password; + } + + @Override + public int grafanaPort() { + return grafanaPort; + } + + @Override + public Duration timeout() { + return Duration.ofMinutes(1); + } +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfig.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfig.java new file mode 100644 index 0000000000000..d6bb59e981d58 --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfig.java @@ -0,0 +1,69 @@ +package io.quarkus.observability.common.config; + +import java.util.Optional; +import java.util.Set; + +import io.smallrye.config.WithDefault; + +public interface ContainerConfig { + + /** + * If DevServices has been explicitly enabled or disabled. DevServices is generally enabled + * by default, unless there is an existing configuration present. + *

+ * When DevServices is enabled Quarkus will attempt to automatically configure and start + * a containers when running in Dev or Test mode and when Docker is running. + */ + @WithDefault("true") + boolean enabled(); + + /** + * The container image name to use, for container based DevServices providers. + */ + String imageName(); + + /** + * Indicates if the container managed by Quarkus Dev Services is shared. + * When shared, Quarkus looks for running containers using label-based service discovery. + * If a matching container is found, it is used, and so a second one is not started. + * Otherwise, Dev Services starts a new container. + *

+ * The discovery uses the {@code quarkus-dev-service-label} label. + * The value is configured using the {@code service-name} property. + *

+ * Container sharing is only used in dev mode. + */ + @WithDefault("true") + boolean shared(); + + /** + * Network aliases. + * + * @return metwork aliases + */ + Optional> networkAliases(); + + /** + * The full name of the label attached to the started container. + * This label is used when {@code shared} is set to {@code true}. + * In this case, before starting a container, Dev Services for looks for a container with th label + * set to the configured value. If found, it will use this container instead of starting a new one. Otherwise, it + * starts a new container with this label set to the specified value. + *

+ * This property is used when you need multiple shared containers. + */ + String label(); + + /** + * The value of the {@code quarkus-dev-service} label attached to the started container. + * This property is used when {@code shared} is set to {@code true}. + * In this case, before starting a container, Dev Services for looks for a container with the + * {@code quarkus-dev-service} label + * set to the configured value. If found, it will use this container instead of starting a new one. Otherwise, it + * starts a new container with the {@code quarkus-dev-service} label set to the specified value. + *

+ * This property is used when you need multiple shared containers. + */ + @WithDefault("quarkus") + String serviceName(); +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfigUtil.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfigUtil.java new file mode 100644 index 0000000000000..8d3ce6d1a0b42 --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfigUtil.java @@ -0,0 +1,41 @@ +package io.quarkus.observability.common.config; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Objects; + +public class ContainerConfigUtil { + /** + * We need a per config method equals, + * so that we know when the config changes. + */ + public static boolean isEqual(ContainerConfig cc1, ContainerConfig cc2) { + Class c1 = cc1.getClass(); + Class c2 = cc1.getClass(); + if (!c1.equals(c2)) { + return false; + } + + Class i = Arrays.stream(c1.getInterfaces()) + .filter(ContainerConfig.class::isAssignableFrom) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Missing ContainerConfig based interface")); + Method[] methods = i.getMethods(); // should get all config methods + for (Method m : methods) { + Object v1 = invoke(m, cc1); + Object v2 = invoke(m, cc2); + if (!Objects.equals(v1, v2)) { + return false; + } + } + return true; + } + + private static Object invoke(Method m, Object target) { + try { + return m.invoke(target); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/GrafanaConfig.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/GrafanaConfig.java new file mode 100644 index 0000000000000..229522ab8e027 --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/GrafanaConfig.java @@ -0,0 +1,33 @@ +package io.quarkus.observability.common.config; + +import java.time.Duration; + +import io.smallrye.config.WithDefault; + +public interface GrafanaConfig extends ContainerConfig { + + // copied from ContainerConfig, config hierarchy workaround + + @WithDefault("true") + boolean enabled(); + + @WithDefault("true") + boolean shared(); + + @WithDefault("quarkus") + String serviceName(); + + // --- + + @WithDefault("admin") + String username(); + + @WithDefault("admin") + String password(); + + @WithDefault("3000") + int grafanaPort(); + + @WithDefault("PT1M") + Duration timeout(); +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/LgtmConfig.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/LgtmConfig.java new file mode 100644 index 0000000000000..ae7a239cb647f --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/LgtmConfig.java @@ -0,0 +1,23 @@ +package io.quarkus.observability.common.config; + +import java.util.Optional; +import java.util.Set; + +import io.quarkus.observability.common.ContainerConstants; +import io.quarkus.runtime.annotations.ConfigGroup; +import io.smallrye.config.WithDefault; + +@ConfigGroup +public interface LgtmConfig extends GrafanaConfig { + @WithDefault(ContainerConstants.LGTM) + String imageName(); + + @WithDefault("lgtm,lgtm.testcontainer.docker") + Optional> networkAliases(); + + @WithDefault("quarkus-dev-service-lgtm") + String label(); + + @WithDefault("4318") + int otlpPort(); +} diff --git a/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ModulesConfiguration.java b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ModulesConfiguration.java new file mode 100644 index 0000000000000..7625f622234ce --- /dev/null +++ b/extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ModulesConfiguration.java @@ -0,0 +1,8 @@ +package io.quarkus.observability.common.config; + +import io.quarkus.runtime.annotations.ConfigDocSection; + +public interface ModulesConfiguration { + @ConfigDocSection + LgtmConfig lgtm(); +} diff --git a/extensions/observability-devservices/deployment/pom.xml b/extensions/observability-devservices/deployment/pom.xml new file mode 100644 index 0000000000000..b2d7788a0da5c --- /dev/null +++ b/extensions/observability-devservices/deployment/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + quarkus-observability-devservices-parent + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-devservices-deployment + Quarkus - Observability Dev Services - Deployment + + + + io.quarkus + quarkus-core-deployment + + + io.quarkus + quarkus-devservices-deployment + + + io.quarkus + quarkus-kubernetes-spi + + + io.quarkus + quarkus-observability-devservices + + + + io.quarkus + quarkus-junit5-internal + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.assertj + assertj-core + test + + + org.awaitility + awaitility + test + + + io.rest-assured + rest-assured + test + + + + + + + maven-compiler-plugin + + + + io.quarkus + quarkus-extension-processor + ${project.version} + + + + + + + diff --git a/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/DevResourcesBuildItem.java b/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/DevResourcesBuildItem.java new file mode 100644 index 0000000000000..4bde7af481a59 --- /dev/null +++ b/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/DevResourcesBuildItem.java @@ -0,0 +1,6 @@ +package io.quarkus.observability.deployment; + +import io.quarkus.builder.item.SimpleBuildItem; + +final class DevResourcesBuildItem extends SimpleBuildItem { +} diff --git a/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/DevResourcesProcessor.java b/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/DevResourcesProcessor.java new file mode 100644 index 0000000000000..88d17c7c34d13 --- /dev/null +++ b/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/DevResourcesProcessor.java @@ -0,0 +1,51 @@ +package io.quarkus.observability.deployment; + +import java.util.function.BooleanSupplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.quarkus.deployment.IsNormal; +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.deployment.annotations.BuildSteps; +import io.quarkus.deployment.annotations.ExecutionTime; +import io.quarkus.deployment.annotations.Record; +import io.quarkus.deployment.builditem.FeatureBuildItem; +import io.quarkus.deployment.builditem.RunTimeConfigBuilderBuildItem; +import io.quarkus.deployment.builditem.ShutdownContextBuildItem; +import io.quarkus.observability.runtime.DevResourceShutdownRecorder; +import io.quarkus.observability.runtime.DevResourcesConfigBuilder; +import io.quarkus.observability.runtime.config.ObservabilityConfiguration; + +@BuildSteps(onlyIfNot = IsNormal.class, onlyIf = DevResourcesProcessor.IsEnabled.class) +class DevResourcesProcessor { + private static final Logger log = LoggerFactory.getLogger(DevResourcesProcessor.class); + private static final String FEATURE = "devresources"; + + @BuildStep + FeatureBuildItem feature() { + return new FeatureBuildItem(FEATURE); + } + + @BuildStep + public RunTimeConfigBuilderBuildItem registerDevResourcesConfigSource() { + log.info("Adding dev resources config builder"); + return new RunTimeConfigBuilderBuildItem(DevResourcesConfigBuilder.class); + } + + @BuildStep + @Record(ExecutionTime.RUNTIME_INIT) + public DevResourcesBuildItem shutdownDevResources(DevResourceShutdownRecorder recorder, ShutdownContextBuildItem shutdown) { + recorder.shutdown(shutdown); + return new DevResourcesBuildItem(); + } + + public static class IsEnabled implements BooleanSupplier { + ObservabilityConfiguration config; + + public boolean getAsBoolean() { + return config.devResources() && !config.enabled(); + } + } + +} diff --git a/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/ObservabilityDevServiceProcessor.java b/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/ObservabilityDevServiceProcessor.java new file mode 100644 index 0000000000000..48c5932448d18 --- /dev/null +++ b/extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/ObservabilityDevServiceProcessor.java @@ -0,0 +1,216 @@ +package io.quarkus.observability.deployment; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BooleanSupplier; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.jboss.logging.Logger; + +import io.quarkus.deployment.Feature; +import io.quarkus.deployment.IsNormal; +import io.quarkus.deployment.annotations.BuildProducer; +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.deployment.annotations.BuildSteps; +import io.quarkus.deployment.builditem.CuratedApplicationShutdownBuildItem; +import io.quarkus.deployment.builditem.DevServicesResultBuildItem; +import io.quarkus.deployment.builditem.DockerStatusBuildItem; +import io.quarkus.deployment.builditem.FeatureBuildItem; +import io.quarkus.deployment.builditem.LaunchModeBuildItem; +import io.quarkus.deployment.console.ConsoleInstalledBuildItem; +import io.quarkus.deployment.console.StartupLogCompressor; +import io.quarkus.deployment.dev.devservices.GlobalDevServicesConfig; +import io.quarkus.deployment.logging.LoggingSetupBuildItem; +import io.quarkus.devservices.common.ContainerLocator; +import io.quarkus.observability.common.config.ContainerConfig; +import io.quarkus.observability.common.config.ContainerConfigUtil; +import io.quarkus.observability.common.config.ModulesConfiguration; +import io.quarkus.observability.devresource.Container; +import io.quarkus.observability.devresource.DevResourceLifecycleManager; +import io.quarkus.observability.devresource.DevResources; +import io.quarkus.observability.runtime.config.ObservabilityConfiguration; +import io.quarkus.runtime.LaunchMode; + +@BuildSteps(onlyIfNot = IsNormal.class, onlyIf = { GlobalDevServicesConfig.Enabled.class, + ObservabilityDevServiceProcessor.IsEnabled.class }) +class ObservabilityDevServiceProcessor { + private static final Logger log = Logger.getLogger(ObservabilityDevServiceProcessor.class); + + private static final Map devServices = new ConcurrentHashMap<>(); + private static final Map capturedDevServicesConfigurations = new ConcurrentHashMap<>(); + private static final Map firstStart = new ConcurrentHashMap<>(); + + public static class IsEnabled implements BooleanSupplier { + ObservabilityConfiguration config; + + public boolean getAsBoolean() { + return config.enabled() && !config.devResources(); + } + } + + @BuildStep + FeatureBuildItem feature() { + return new FeatureBuildItem(Feature.OBSERVABILITY); + } + + private String devId(DevResourceLifecycleManager dev) { + String sn = dev.getClass().getSimpleName(); + int p = sn.indexOf("Resource"); + return sn.substring(0, p != -1 ? p : sn.length()); + } + + @BuildStep + public void startContainers(LaunchModeBuildItem launchMode, + DockerStatusBuildItem dockerStatusBuildItem, + ObservabilityConfiguration configuration, + Optional consoleInstalledBuildItem, + CuratedApplicationShutdownBuildItem closeBuildItem, + LoggingSetupBuildItem loggingSetupBuildItem, + GlobalDevServicesConfig devServicesConfig, + BuildProducer services) { + + if (!configuration.enabled()) { + log.infof("Observability dev services are disabled in config"); + return; + } + + if (!dockerStatusBuildItem.isDockerAvailable()) { + log.warn("Please get a working Docker instance"); + return; + } + + @SuppressWarnings("rawtypes") + List resources = DevResources.resources(); + // this should throw an exception on a duplicate + //noinspection ResultOfMethodCallIgnored + resources.stream().collect(Collectors.toMap(this::devId, Function.identity())); + + @SuppressWarnings("rawtypes") + Stream stream = resources.stream(); + if (configuration.parallel()) { + stream = stream.parallel(); + } + + stream.forEach(dev -> { + String devId = devId(dev); + + // only do get, not remove, so it can be re-used + DevServicesResultBuildItem.RunningDevService devService = devServices.get(devId); + ContainerConfig currentDevServicesConfiguration = dev.config(configuration); + + if (devService != null) { + ContainerConfig capturedDevServicesConfiguration = capturedDevServicesConfigurations.get(devId); + boolean equalConfig = ContainerConfigUtil.isEqual(capturedDevServicesConfiguration, + currentDevServicesConfiguration); + if (equalConfig) { + log.debugf("Equal config, re-using existing %s container", devId); + services.produce(devService.toBuildItem()); + return; + } + try { + devService.close(); + } catch (Throwable e) { + log.errorf("Failed to stop %s container", devId, e); + } + } + + devServices.remove(devId); // clean-up + capturedDevServicesConfigurations.put(devId, currentDevServicesConfiguration); + + StartupLogCompressor compressor = new StartupLogCompressor( + (launchMode.isTest() ? "(test) " : "") + devId + " Dev Services Starting:", + consoleInstalledBuildItem, + loggingSetupBuildItem, + s -> false, + s -> s.contains(getClass().getSimpleName())); // log if it comes from this class + try { + DevServicesResultBuildItem.RunningDevService newDevService = startContainer( + devId, + dev, + currentDevServicesConfiguration, + configuration, + devServicesConfig.timeout); + if (newDevService == null) { + compressor.closeAndDumpCaptured(); + return; + } else { + compressor.close(); + } + + devService = newDevService; + devServices.put(devId, newDevService); + } catch (Throwable t) { + compressor.closeAndDumpCaptured(); + throw new RuntimeException(t); + } + + if (firstStart.computeIfAbsent(devId, x -> true)) { + Runnable closeTask = () -> { + DevServicesResultBuildItem.RunningDevService current = devServices.get(devId); + if (current != null) { + try { + current.close(); + } catch (Throwable t) { + log.errorf("Failed to stop %s container", devId, t); + } + } + firstStart.remove(devId); + //noinspection resource + devServices.remove(devId); + capturedDevServicesConfigurations.remove(devId); + }; + closeBuildItem.addCloseTask(closeTask, true); + } + + services.produce(devService.toBuildItem()); + }); + } + + private DevServicesResultBuildItem.RunningDevService startContainer( + String devId, + DevResourceLifecycleManager dev, + ContainerConfig capturedDevServicesConfiguration, + ModulesConfiguration root, + Optional timeout) { + + if (!capturedDevServicesConfiguration.enabled()) { + // explicitly disabled + log.debugf("Not starting Dev Services for %s as it has been disabled in the config", devId); + return null; + } + + if (!dev.enable()) { + return null; + } + + final Supplier defaultContainerSupplier = () -> { + Container container = dev.container(capturedDevServicesConfiguration, root); + timeout.ifPresent(container::withStartupTimeout); + Map config = dev.start(); + log.infof("Dev Service %s started, config: %s", devId, config); + return new DevServicesResultBuildItem.RunningDevService( + Feature.OBSERVABILITY.getName(), container.getContainerId(), + container.closeableCallback(capturedDevServicesConfiguration.serviceName()), config); + }; + + Map config = new LinkedHashMap<>(); // old config + ContainerLocator containerLocator = new ContainerLocator(capturedDevServicesConfiguration.label(), 0); // can be 0, as we don't use it + return containerLocator + .locateContainer( + capturedDevServicesConfiguration.serviceName(), capturedDevServicesConfiguration.shared(), + LaunchMode.current(), (p, ca) -> config.putAll(dev.config(p, ca.getHost(), ca.getPort()))) + .map(cid -> { + log.infof("Dev Service %s re-used, config: %s", devId, config); + return new DevServicesResultBuildItem.RunningDevService(Feature.OBSERVABILITY.getName(), cid, + null, config); + }) + .orElseGet(defaultContainerSupplier); + } +} diff --git a/extensions/observability-devservices/pom.xml b/extensions/observability-devservices/pom.xml new file mode 100644 index 0000000000000..972d9e41e83b7 --- /dev/null +++ b/extensions/observability-devservices/pom.xml @@ -0,0 +1,56 @@ + + + + quarkus-extensions-parent + io.quarkus + 999-SNAPSHOT + ../pom.xml + + 4.0.0 + + quarkus-observability-devservices-parent + Quarkus - Observability Dev Services - Parent + pom + + common + testcontainers + testlibs + deployment + runtime + + sink/lgtm + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce + + + + + classpath:enforcer-rules/quarkus-banned-dependencies.xml + + + classpath:enforcer-rules/quarkus-banned-dependencies-okhttp.xml + + + + + + + + + \ No newline at end of file diff --git a/extensions/observability-devservices/runtime/pom.xml b/extensions/observability-devservices/runtime/pom.xml new file mode 100644 index 0000000000000..3d4b5ec69174e --- /dev/null +++ b/extensions/observability-devservices/runtime/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + + quarkus-observability-devservices-parent + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-devservices + Quarkus - Observability Dev Services - Runtime + Serve and consume Observability Dev Services + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-core + + + + io.quarkus + quarkus-observability-devservices-common + + + io.quarkus + quarkus-observability-devresource-common + + + + + io.quarkus + quarkus-junit5-internal + test + + + org.awaitility + awaitility + test + + + org.assertj + assertj-core + test + + + org.mockito + mockito-core + test + + + + + + + io.quarkus + quarkus-extension-maven-plugin + + + generate-extension-descriptor + + extension-descriptor + + process-resources + + ${project.groupId}:${project.artifactId}-deployment:${project.version} + + + + + + maven-compiler-plugin + + + + io.quarkus + quarkus-extension-processor + ${project.version} + + + + + + + diff --git a/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/DevResourceShutdownRecorder.java b/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/DevResourceShutdownRecorder.java new file mode 100644 index 0000000000000..0d63c09b081dc --- /dev/null +++ b/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/DevResourceShutdownRecorder.java @@ -0,0 +1,12 @@ +package io.quarkus.observability.runtime; + +import io.quarkus.observability.devresource.DevResources; +import io.quarkus.runtime.ShutdownContext; +import io.quarkus.runtime.annotations.Recorder; + +@Recorder +public class DevResourceShutdownRecorder { + public void shutdown(ShutdownContext context) { + context.addLastShutdownTask(DevResources::stop); + } +} diff --git a/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/DevResourcesConfigBuilder.java b/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/DevResourcesConfigBuilder.java new file mode 100644 index 0000000000000..8d327a1d4bb0c --- /dev/null +++ b/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/DevResourcesConfigBuilder.java @@ -0,0 +1,18 @@ +package io.quarkus.observability.runtime; + +import io.quarkus.observability.devresource.DevResourcesConfigSource; +import io.quarkus.runtime.configuration.ConfigBuilder; +import io.smallrye.config.SmallRyeConfigBuilder; + +public class DevResourcesConfigBuilder implements ConfigBuilder { + @Override + public SmallRyeConfigBuilder configBuilder(SmallRyeConfigBuilder builder) { + return builder.withSources(new DevResourcesConfigSource()); + } + + @Override + public int priority() { + // greater than any default Microprofile ConfigSource + return 500; + } +} diff --git a/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/config/ObservabilityConfiguration.java b/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/config/ObservabilityConfiguration.java new file mode 100644 index 0000000000000..e9327be656c1e --- /dev/null +++ b/extensions/observability-devservices/runtime/src/main/java/io/quarkus/observability/runtime/config/ObservabilityConfiguration.java @@ -0,0 +1,35 @@ +package io.quarkus.observability.runtime.config; + +import io.quarkus.observability.common.config.ModulesConfiguration; +import io.quarkus.runtime.annotations.ConfigPhase; +import io.quarkus.runtime.annotations.ConfigRoot; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; + +@ConfigMapping(prefix = "quarkus.observability") +@ConfigRoot(phase = ConfigPhase.BUILD_TIME) +public interface ObservabilityConfiguration extends ModulesConfiguration { + /** + * If DevServices has been explicitly enabled or disabled. DevServices is generally enabled + * by default, unless there is an existing configuration present. + *

+ * When DevServices is enabled Quarkus will attempt to automatically configure and start + * a containers when running in Dev or Test mode and when Docker is running. + */ + @WithDefault("true") + boolean enabled(); + + /** + * Enable simplified usage of dev resources, + * instead of full observability processing. + * Make sure @code{enabled} is set to false. + */ + @WithDefault("false") + boolean devResources(); + + /** + * Do we start the dev services in parallel. + */ + @WithDefault("false") + boolean parallel(); +} diff --git a/extensions/observability-devservices/sink/lgtm/pom.xml b/extensions/observability-devservices/sink/lgtm/pom.xml new file mode 100644 index 0000000000000..467432630e33d --- /dev/null +++ b/extensions/observability-devservices/sink/lgtm/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + quarkus-observability-devservices-parent + io.quarkus + 999-SNAPSHOT + ../../pom.xml + + + quarkus-observability-devservices-lgtm + Quarkus - Observability Dev Services - LGTM + + + + io.quarkus + quarkus-observability-devservices + + + io.quarkus + quarkus-observability-devresource-lgtm + + + diff --git a/extensions/observability-devservices/testcontainers/pom.xml b/extensions/observability-devservices/testcontainers/pom.xml new file mode 100644 index 0000000000000..5f4a15c826d64 --- /dev/null +++ b/extensions/observability-devservices/testcontainers/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + + quarkus-observability-devservices-parent + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-testcontainers + Quarkus - Observability Dev Services - Testcontainers + + + + io.quarkus + quarkus-devservices-common + + + io.quarkus + quarkus-observability-devservices-common + + + org.testcontainers + testcontainers + + + junit + junit + + + + + io.quarkus + quarkus-junit4-mock + + + org.junit.jupiter + junit-jupiter-api + test + + + + \ No newline at end of file diff --git a/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/GrafanaContainer.java b/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/GrafanaContainer.java new file mode 100644 index 0000000000000..0218c6911b3cf --- /dev/null +++ b/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/GrafanaContainer.java @@ -0,0 +1,35 @@ +package io.quarkus.observability.testcontainers; + +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategy; + +import io.quarkus.observability.common.config.GrafanaConfig; + +@SuppressWarnings("resource") +public abstract class GrafanaContainer, C extends GrafanaConfig> + extends ObservabilityContainer { + protected static final String DATASOURCES_PATH = "/etc/grafana/provisioning/datasources/custom.yaml"; + + protected C config; + + public GrafanaContainer(C config) { + super(config); + this.config = config; + withEnv("GF_SECURITY_ADMIN_USER", config.username()); + withEnv("GF_SECURITY_ADMIN_PASSWORD", config.password()); + addExposedPort(config.grafanaPort()); + waitingFor(grafanaWaitStrategy()); + } + + public int getGrafanaPort() { + return getMappedPort(config.grafanaPort()); + } + + private WaitStrategy grafanaWaitStrategy() { + return new HttpWaitStrategy() + .forPath("/") + .forPort(config.grafanaPort()) + .forStatusCode(200) + .withStartupTimeout(config.timeout()); + } +} diff --git a/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/LgtmContainer.java b/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/LgtmContainer.java new file mode 100644 index 0000000000000..fd31bab3be9c0 --- /dev/null +++ b/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/LgtmContainer.java @@ -0,0 +1,45 @@ +package io.quarkus.observability.testcontainers; + +import java.util.Optional; +import java.util.Set; + +import io.quarkus.observability.common.ContainerConstants; +import io.quarkus.observability.common.config.AbstractGrafanaConfig; +import io.quarkus.observability.common.config.LgtmConfig; + +public class LgtmContainer extends GrafanaContainer { + protected static final String LGTM_NETWORK_ALIAS = "ltgm.testcontainer.docker"; + + public LgtmContainer() { + this(new LgtmConfigImpl()); + } + + public LgtmContainer(LgtmConfig config) { + super(config); + addExposedPorts(config.otlpPort()); + } + + public int getOtlpPort() { + return getMappedPort(config.otlpPort()); + } + + protected static class LgtmConfigImpl extends AbstractGrafanaConfig implements LgtmConfig { + public LgtmConfigImpl() { + this(ContainerConstants.LGTM); + } + + public LgtmConfigImpl(String imageName) { + super(imageName); + } + + @Override + public Optional> networkAliases() { + return Optional.of(Set.of("lgtm", LGTM_NETWORK_ALIAS)); + } + + @Override + public int otlpPort() { + return ContainerConstants.OTEL_HTTP_EXPORTER_PORT; + } + } +} diff --git a/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/ObservabilityContainer.java b/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/ObservabilityContainer.java new file mode 100644 index 0000000000000..bcc761dddd58a --- /dev/null +++ b/extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/ObservabilityContainer.java @@ -0,0 +1,72 @@ +package io.quarkus.observability.testcontainers; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.DockerImageName; + +import io.quarkus.observability.common.config.ContainerConfig; + +@SuppressWarnings("resource") +public abstract class ObservabilityContainer, C extends ContainerConfig> + extends GenericContainer { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final Logger dockerLog = LoggerFactory.getLogger(getClass().getName() + ".docker"); + + public ObservabilityContainer(C config) { + super(DockerImageName.parse(config.imageName())); + withLogConsumer(frameConsumer()); + withLabel(config.label(), config.serviceName()); + Optional> aliases = config.networkAliases(); + aliases.map(s -> s.toArray(new String[0])).ifPresent(this::withNetworkAliases); + if (config.shared()) { + withNetwork(Network.SHARED); + } + } + + protected Consumer frameConsumer() { + return frame -> logger().debug(frame.getUtf8String().stripTrailing()); + } + + protected byte[] getResourceAsBytes(String resource) { + try (InputStream in = getClass().getClassLoader().getResourceAsStream(resource)) { + return in.readAllBytes(); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } + } + + @SuppressWarnings("OctalInteger") + protected void addFileToContainer(byte[] content, String pathInContainer) { + logger().info("Content [{}]: \n{}", pathInContainer, new String(content, StandardCharsets.UTF_8)); + copyFileToContainer(Transferable.of(content, 0777), pathInContainer); + } + + @Override + protected Logger logger() { + return dockerLog; + } + + @Override + public void start() { + log.info("Starting {} ...", getClass().getSimpleName()); + super.start(); + } + + @Override + public void stop() { + log.info("Stopping {}...", getClass().getSimpleName()); + super.stop(); + } +} diff --git a/extensions/observability-devservices/testlibs/devresource-common/pom.xml b/extensions/observability-devservices/testlibs/devresource-common/pom.xml new file mode 100644 index 0000000000000..3185155b20277 --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-common/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + quarkus-observability-testlibs + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-devresource-common + Quarkus - Observability Dev Services - Dev Resource Common + + + + org.eclipse.microprofile.config + microprofile-config-api + + + org.jboss.logging + jboss-logging + + + io.quarkus + quarkus-observability-devservices-common + + + io.quarkus + quarkus-test-common + + + io.quarkus + quarkus-core-deployment + + + + + \ No newline at end of file diff --git a/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/Container.java b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/Container.java new file mode 100644 index 0000000000000..e591fb9555a51 --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/Container.java @@ -0,0 +1,21 @@ +package io.quarkus.observability.devresource; + +import java.io.Closeable; +import java.time.Duration; + +import io.quarkus.observability.common.config.ContainerConfig; + +/** + * Simple container abstraction, e.g. similar to GenericContainer + */ +public interface Container { + void start(); + + void stop(); + + String getContainerId(); + + void withStartupTimeout(Duration duration); + + Closeable closeableCallback(String serviceName); +} diff --git a/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResourceLifecycleManager.java b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResourceLifecycleManager.java new file mode 100644 index 0000000000000..be32e44084607 --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResourceLifecycleManager.java @@ -0,0 +1,83 @@ +package io.quarkus.observability.devresource; + +import java.util.Map; + +import io.quarkus.observability.common.config.ContainerConfig; +import io.quarkus.observability.common.config.ModulesConfiguration; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; + +/** + * Extends {@link io.quarkus.test.common.QuarkusTestResourceLifecycleManager} + * so that classes implement both interfaces at the same time - simplifying testing. + */ +public interface DevResourceLifecycleManager extends QuarkusTestResourceLifecycleManager { + + // Put order constants here -- order by dependency + + int METRICS = 5000; + int SCRAPER = 7500; + int GRAFANA = 10000; + int JAEGER = 20000; + int OTEL = 20000; + + //---- + + /** + * Get resource's config from main observability configuration. + * + * @param configuration main observability configuration + * @return module's config + */ + T config(ModulesConfiguration configuration); + + /** + * Should we enable / start this dev resource. + * e.g. we could already have actual service running + * Each impl should provide its own reason on why it disabled dev service. + * + * @return true if ok to start new dev service, false otherwise + */ + default boolean enable() { + return true; + } + + /** + * Create container from config. + * + * @param config the config + * @return container id + */ + default Container container(T config) { + throw new IllegalStateException("Should be implemented!"); + } + + /** + * Create container from config. + * + * @param config the config + * @param root the all modules config + * @return container id + */ + default Container container(T config, ModulesConfiguration root) { + return container(config); + } + + /** + * Deduct current config from params. + * If port are too dynamic / configured, it's hard to deduct, + * since configuration is not part of the devservice state. + * e.g. different ports then usual - Grafana UI is 3000, if you do not use 3000, + * it's hard or impossible to know which port belongs to certain property. + * + * @return A map of system properties that should be set for the running dev-mode app + */ + Map config(int privatePort, String host, int publicPort); + + /** + * Called even before {@link #start()} so that the implementation can prepare itself + * to be used as dev resource (as opposed to test resource which uses a different + * init() method). + */ + default void initDev() { + } +} diff --git a/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResources.java b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResources.java new file mode 100644 index 0000000000000..0dbd79ed0579e --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResources.java @@ -0,0 +1,88 @@ +package io.quarkus.observability.devresource; + +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.stream.Collectors; + +import org.jboss.logging.Logger; + +/** + * A registry of dev resources. + */ +@SuppressWarnings("rawtypes") +public class DevResources { + private static final Logger log = Logger.getLogger(DevResources.class); + + private static List resources; + private static Map map; + + /** + * @return list of found dev resources. + */ + public static synchronized List resources() { + if (resources == null) { + log.info("Activating dev resources"); + + resources = ServiceLoader + .load(DevResourceLifecycleManager.class, Thread.currentThread().getContextClassLoader()) + .stream() + .map(ServiceLoader.Provider::get) + .sorted(Comparator.comparing(DevResourceLifecycleManager::order)) + .collect(Collectors.toList()); + + log.infof("Found dev resources: %s", resources); + } + return resources; + } + + /** + * Ensures all dev resources are started and returns a map of config properties. + * + * @return a map of config properties to be returned by {@link DevResourcesConfigSource} + */ + static synchronized Map ensureStarted() { + if (map == null) { + try { + for (var res : resources()) { + res.initDev(); + } + } catch (Exception e) { + log.error("Exception initializing dev resource manager", e); + throw e; + } + try { + var map = new HashMap(); + for (var res : resources()) { + var resMap = res.start(); + log.infof("Dev resource [%s] contributed config: %s", res.getClass().getSimpleName(), resMap); + map.putAll(resMap); + } + DevResources.map = Collections.unmodifiableMap(map); + } catch (Exception e) { + log.error("Exception starting dev resource", e); + throw e; + } + } + return map; + } + + /** + * Stops all dev resources. + */ + public static synchronized void stop() { + if (map != null) { + for (var i = resources().listIterator(resources().size()); i.hasPrevious();) { + try { + i.previous().stop(); + } catch (Exception e) { + log.warn("Exception stopping dev resource", e); + } + } + map = null; + } + } +} diff --git a/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResourcesConfigSource.java b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResourcesConfigSource.java new file mode 100644 index 0000000000000..b3458c5be2be2 --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/DevResourcesConfigSource.java @@ -0,0 +1,28 @@ +package io.quarkus.observability.devresource; + +import java.util.Set; + +import org.eclipse.microprofile.config.spi.ConfigSource; + +public class DevResourcesConfigSource implements ConfigSource { + @Override + public Set getPropertyNames() { + return DevResources.ensureStarted().keySet(); + } + + @Override + public String getValue(String propertyName) { + return DevResources.ensureStarted().get(propertyName); + } + + @Override + public String getName() { + return "DevResourcesConfigSource"; + } + + @Override + public int getOrdinal() { + // greater than any default Microprofile ConfigSource + return 500; + } +} diff --git a/extensions/observability-devservices/testlibs/devresource-lgtm/pom.xml b/extensions/observability-devservices/testlibs/devresource-lgtm/pom.xml new file mode 100644 index 0000000000000..9a89f891abe9b --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-lgtm/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + quarkus-observability-testlibs + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-devresource-lgtm + Quarkus - Observability Dev Services - OTel LGTM Dev Resource + + + + io.quarkus + quarkus-observability-devresource-testcontainers + + + io.quarkus + quarkus-observability-testcontainers + + + + \ No newline at end of file diff --git a/extensions/observability-devservices/testlibs/devresource-lgtm/src/main/java/io/quarkus/observability/devresource/lgtm/LgtmResource.java b/extensions/observability-devservices/testlibs/devresource-lgtm/src/main/java/io/quarkus/observability/devresource/lgtm/LgtmResource.java new file mode 100644 index 0000000000000..6273380d805ba --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-lgtm/src/main/java/io/quarkus/observability/devresource/lgtm/LgtmResource.java @@ -0,0 +1,54 @@ +package io.quarkus.observability.devresource.lgtm; + +import java.util.Map; + +import io.quarkus.observability.common.ContainerConstants; +import io.quarkus.observability.common.config.LgtmConfig; +import io.quarkus.observability.common.config.ModulesConfiguration; +import io.quarkus.observability.devresource.Container; +import io.quarkus.observability.devresource.DevResourceLifecycleManager; +import io.quarkus.observability.devresource.testcontainers.ContainerResource; +import io.quarkus.observability.testcontainers.LgtmContainer; + +public class LgtmResource extends ContainerResource { + + @Override + public LgtmConfig config(ModulesConfiguration configuration) { + return configuration.lgtm(); + } + + @Override + public Container container(LgtmConfig config, ModulesConfiguration root) { + return set(new LgtmContainer(config)); + } + + @Override + public Map config(int privatePort, String host, int publicPort) { + switch (privatePort) { + case ContainerConstants.GRAFANA_PORT: + return Map.of("quarkus.grafana.url", String.format("%s:%s", host, publicPort)); + case ContainerConstants.OTEL_GRPC_EXPORTER_PORT: + case ContainerConstants.OTEL_HTTP_EXPORTER_PORT: + return Map.of("quarkus.otel-collector.url", String.format("%s:%s", host, publicPort)); + } + return Map.of(); + } + + @Override + protected LgtmContainer defaultContainer() { + return new LgtmContainer(); + } + + @Override + public Map doStart() { + String host = container.getHost(); + return Map.of( + "quarkus.grafana.url", String.format("%s:%s", host, container.getGrafanaPort()), + "quarkus.otel-collector.url", String.format("%s:%s", host, container.getOtlpPort())); + } + + @Override + public int order() { + return DevResourceLifecycleManager.GRAFANA; + } +} diff --git a/extensions/observability-devservices/testlibs/devresource-lgtm/src/main/resources/META-INF/services/io.quarkus.observability.devresource.DevResourceLifecycleManager b/extensions/observability-devservices/testlibs/devresource-lgtm/src/main/resources/META-INF/services/io.quarkus.observability.devresource.DevResourceLifecycleManager new file mode 100644 index 0000000000000..73702014d3425 --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-lgtm/src/main/resources/META-INF/services/io.quarkus.observability.devresource.DevResourceLifecycleManager @@ -0,0 +1 @@ +io.quarkus.observability.devresource.lgtm.LgtmResource diff --git a/extensions/observability-devservices/testlibs/devresource-testcontainers/pom.xml b/extensions/observability-devservices/testlibs/devresource-testcontainers/pom.xml new file mode 100644 index 0000000000000..1f943c5afaeeb --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-testcontainers/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + quarkus-observability-testlibs + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-devresource-testcontainers + Quarkus - Observability Dev Services - Dev Resource Testcontainers + + + + io.quarkus + quarkus-devservices-common + + + io.quarkus + quarkus-observability-devresource-common + + + org.testcontainers + testcontainers + + + junit + junit + + + + + io.quarkus + quarkus-junit4-mock + + + \ No newline at end of file diff --git a/extensions/observability-devservices/testlibs/devresource-testcontainers/src/main/java/io/quarkus/observability/devresource/testcontainers/ContainerResource.java b/extensions/observability-devservices/testlibs/devresource-testcontainers/src/main/java/io/quarkus/observability/devresource/testcontainers/ContainerResource.java new file mode 100644 index 0000000000000..5ad400f90a920 --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-testcontainers/src/main/java/io/quarkus/observability/devresource/testcontainers/ContainerResource.java @@ -0,0 +1,45 @@ +package io.quarkus.observability.devresource.testcontainers; + +import java.util.Map; + +import org.testcontainers.containers.GenericContainer; + +import io.quarkus.observability.common.config.ContainerConfig; +import io.quarkus.observability.devresource.Container; +import io.quarkus.observability.devresource.DevResourceLifecycleManager; + +/** + * A container resource abstraction + */ +public abstract class ContainerResource, C extends ContainerConfig> + implements DevResourceLifecycleManager { + + protected T container; + protected Container wrapper; + + protected Container set(T container) { + this.container = container; + this.wrapper = new TestcontainerContainer<>(container); + return this.wrapper; + } + + @Override + public Map start() { + if (container == null) { + set(defaultContainer()); + } + container.start(); + return doStart(); + } + + @Override + public void stop() { + if (container != null) { + container.stop(); + } + } + + protected abstract T defaultContainer(); + + protected abstract Map doStart(); +} diff --git a/extensions/observability-devservices/testlibs/devresource-testcontainers/src/main/java/io/quarkus/observability/devresource/testcontainers/TestcontainerContainer.java b/extensions/observability-devservices/testlibs/devresource-testcontainers/src/main/java/io/quarkus/observability/devresource/testcontainers/TestcontainerContainer.java new file mode 100644 index 0000000000000..9799d3a4a4cac --- /dev/null +++ b/extensions/observability-devservices/testlibs/devresource-testcontainers/src/main/java/io/quarkus/observability/devresource/testcontainers/TestcontainerContainer.java @@ -0,0 +1,47 @@ +package io.quarkus.observability.devresource.testcontainers; + +import java.io.Closeable; +import java.time.Duration; +import java.util.Objects; + +import org.testcontainers.containers.GenericContainer; + +import io.quarkus.devservices.common.ContainerShutdownCloseable; +import io.quarkus.observability.common.config.ContainerConfig; +import io.quarkus.observability.devresource.Container; + +/** + * Container impl / wrapper for Testcontainer's GenericContainer + */ +public class TestcontainerContainer, T extends ContainerConfig> implements Container { + private final GenericContainer container; + + public TestcontainerContainer(GenericContainer container) { + this.container = Objects.requireNonNull(container); + } + + @Override + public void start() { + container.start(); + } + + @Override + public void stop() { + container.stop(); + } + + @Override + public String getContainerId() { + return container.getContainerId(); + } + + @Override + public void withStartupTimeout(Duration duration) { + container.withStartupTimeout(duration); + } + + @Override + public Closeable closeableCallback(String serviceName) { + return new ContainerShutdownCloseable(container, serviceName); + } +} diff --git a/extensions/observability-devservices/testlibs/pom.xml b/extensions/observability-devservices/testlibs/pom.xml new file mode 100644 index 0000000000000..a7c4b4df2445e --- /dev/null +++ b/extensions/observability-devservices/testlibs/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + + quarkus-observability-devservices-parent + io.quarkus + 999-SNAPSHOT + + + quarkus-observability-testlibs + pom + Quarkus - Observability Dev Services - Test Libraries + + + devresource-common + devresource-testcontainers + devresource-lgtm + + + \ No newline at end of file diff --git a/extensions/pom.xml b/extensions/pom.xml index ccd152cf03acc..cbd5443006e62 100644 --- a/extensions/pom.xml +++ b/extensions/pom.xml @@ -52,6 +52,7 @@ micrometer-registry-prometheus opentelemetry info + observability-devservices resteasy-classic diff --git a/integration-tests/observability-lgtm/pom.xml b/integration-tests/observability-lgtm/pom.xml new file mode 100644 index 0000000000000..0abc8789677cb --- /dev/null +++ b/integration-tests/observability-lgtm/pom.xml @@ -0,0 +1,121 @@ + + + 4.0.0 + + + quarkus-integration-tests-parent + io.quarkus + 999-SNAPSHOT + + + quarkus-integration-test-observability-lgtm + Quarkus - Integration Tests - Observability LGTM + + + + io.quarkus + quarkus-observability-devservices-lgtm + + + io.quarkus + quarkus-rest + + + + io.quarkiverse.micrometer.registry + quarkus-micrometer-registry-otlp + 3.2.4 + + + io.quarkus + quarkus-micrometer + + + io.quarkus + quarkus-opentelemetry + + + com.fasterxml.jackson.core + jackson-databind + + + io.quarkus + quarkus-junit5 + test + + + io.rest-assured + rest-assured + test + + + org.awaitility + awaitility + test + + + org.assertj + assertj-core + test + + + + + io.quarkus + quarkus-rest-deployment + ${project.version} + pom + test + + + * + * + + + + + io.quarkus + quarkus-opentelemetry-deployment + ${project.version} + pom + test + + + * + * + + + + + io.quarkus + quarkus-micrometer-deployment + ${project.version} + pom + test + + + * + * + + + + + + + + + io.quarkus + quarkus-maven-plugin + + + + build + + + + + + + diff --git a/integration-tests/observability-lgtm/src/main/java/io/quarkus/observability/example/SimpleEndpoint.java b/integration-tests/observability-lgtm/src/main/java/io/quarkus/observability/example/SimpleEndpoint.java new file mode 100644 index 0000000000000..17684efd1e944 --- /dev/null +++ b/integration-tests/observability-lgtm/src/main/java/io/quarkus/observability/example/SimpleEndpoint.java @@ -0,0 +1,48 @@ +package io.quarkus.observability.example; + +import java.security.SecureRandom; +import java.util.Random; + +import jakarta.annotation.PostConstruct; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; + +import org.jboss.logging.Logger; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +@Path("/api") +public class SimpleEndpoint { + private static final Logger log = Logger.getLogger(SimpleEndpoint.class); + + @Inject + MeterRegistry registry; + + Random random = new SecureRandom(); + double[] arr = new double[1]; + + @PostConstruct + public void start() { + String key = System.getProperty("tag-key", "test"); + Gauge.builder("xvalue", arr, a -> arr[0]) + .baseUnit("X") + .description("Some random x") + .tag(key, "x") + .register(registry); + } + + @GET + @Produces(MediaType.TEXT_PLAIN) + @Path("/poke") + public String poke(@QueryParam("f") int f) { + log.infof("Poke %s", f); + double x = random.nextDouble() * f; + arr[0] = x; + return "poke:" + x; + } +} diff --git a/integration-tests/observability-lgtm/src/main/resources/application.properties b/integration-tests/observability-lgtm/src/main/resources/application.properties new file mode 100644 index 0000000000000..b88a2d3f9bb17 --- /dev/null +++ b/integration-tests/observability-lgtm/src/main/resources/application.properties @@ -0,0 +1,17 @@ +quarkus.log.category."io.quarkus.observability".level=DEBUG +quarkus.log.category."io.quarkus.devservices".level=DEBUG + +#micrometer +quarkus.micrometer.export.otlp.enabled=true +quarkus.micrometer.export.otlp.publish=true +quarkus.micrometer.export.otlp.step=PT5S +quarkus.micrometer.export.otlp.default-registry=true +%dev.quarkus.micrometer.export.otlp.url=http://${quarkus.otel-collector.url}/v1/metrics +%prod.quarkus.micrometer.export.otlp.url=http://localhost:4318/v1/metrics + +#opentelemetry +quarkus.otel.exporter.otlp.traces.protocol=http/protobuf +%dev.quarkus.otel.exporter.otlp.traces.endpoint=http://${quarkus.otel-collector.url} +%prod.quarkus.otel.exporter.otlp.traces.endpoint=http://localhost:4318 + +#quarkus.observability.lgtm.image-name=grafana/otel-lgtm diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmLifecycleTest.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmLifecycleTest.java new file mode 100644 index 0000000000000..8660c82c95b93 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmLifecycleTest.java @@ -0,0 +1,17 @@ +package io.quarkus.observability.test; + +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import io.quarkus.observability.devresource.lgtm.LgtmResource; +import io.quarkus.observability.test.support.QuarkusTestResourceTestProfile; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; + +@QuarkusTest +@QuarkusTestResource(value = LgtmResource.class, restrictToAnnotatedClass = true) +@TestProfile(QuarkusTestResourceTestProfile.class) +@DisabledOnOs(OS.WINDOWS) +public class LgtmLifecycleTest extends LgtmTestBase { +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmResourcesIT.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmResourcesIT.java new file mode 100644 index 0000000000000..fe869cf62b010 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmResourcesIT.java @@ -0,0 +1,11 @@ +package io.quarkus.observability.test; + +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import io.quarkus.test.junit.QuarkusIntegrationTest; + +@QuarkusIntegrationTest +@DisabledOnOs(OS.WINDOWS) +public class LgtmResourcesIT extends LgtmResourcesTest { +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmResourcesTest.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmResourcesTest.java new file mode 100644 index 0000000000000..6cd232235b38f --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmResourcesTest.java @@ -0,0 +1,14 @@ +package io.quarkus.observability.test; + +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import io.quarkus.observability.test.support.DevResourcesTestProfile; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; + +@QuarkusTest +@TestProfile(DevResourcesTestProfile.class) +@DisabledOnOs(OS.WINDOWS) +public class LgtmResourcesTest extends LgtmTestBase { +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmServicesTest.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmServicesTest.java new file mode 100644 index 0000000000000..33b7e0c13da7c --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmServicesTest.java @@ -0,0 +1,11 @@ +package io.quarkus.observability.test; + +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import io.quarkus.test.junit.QuarkusTest; + +@QuarkusTest +@DisabledOnOs(OS.WINDOWS) +public class LgtmServicesTest extends LgtmTestBase { +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmTestBase.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmTestBase.java new file mode 100644 index 0000000000000..6223f25763584 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/LgtmTestBase.java @@ -0,0 +1,33 @@ +package io.quarkus.observability.test; + +import java.util.concurrent.TimeUnit; + +import org.awaitility.Awaitility; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; +import org.junit.jupiter.api.Test; + +import io.quarkus.observability.test.support.GrafanaClient; +import io.restassured.RestAssured; + +public abstract class LgtmTestBase { + private final Logger log = Logger.getLogger(getClass()); + + @ConfigProperty(name = "quarkus.grafana.url") + String url; + + @Test + public void testTracing() { + log.info("Testing Grafana ..."); + String response = RestAssured.get("/api/poke?f=100").body().asString(); + log.info("Response: " + response); + GrafanaClient client = new GrafanaClient("http://" + url, "admin", "admin"); + Awaitility.await().atMost(61, TimeUnit.SECONDS).until( + client::user, + u -> "admin".equals(u.login)); + Awaitility.await().atMost(61, TimeUnit.SECONDS).until( + () -> client.query("xvalue_X"), + result -> !result.data.result.isEmpty()); + } + +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/DevResourcesTestProfile.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/DevResourcesTestProfile.java new file mode 100644 index 0000000000000..1110a11073098 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/DevResourcesTestProfile.java @@ -0,0 +1,14 @@ +package io.quarkus.observability.test.support; + +import java.util.Map; + +import io.quarkus.test.junit.QuarkusTestProfile; + +public class DevResourcesTestProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.observability.dev-resources", "true", + "quarkus.observability.enabled", "false"); + } +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/GrafanaClient.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/GrafanaClient.java new file mode 100644 index 0000000000000..40f31cc689589 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/GrafanaClient.java @@ -0,0 +1,92 @@ +package io.quarkus.observability.test.support; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Base64; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class GrafanaClient { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final String url; + private final String username; + private final String password; + + public GrafanaClient(String url, String username, String password) { + this.url = url; + this.username = username; + this.password = password; + } + + private void handle( + String path, + Function method, + HttpResponse.BodyHandler bodyHandler, + BiConsumer, T> consumer) { + try { + String credentials = username + ":" + password; + String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes()); + + HttpClient httpClient = HttpClient.newHttpClient(); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .uri(URI.create(url + path)) + .header("Authorization", "Basic " + encodedCredentials); + HttpRequest request = method.apply(builder).build(); + + HttpResponse response = httpClient.send(request, bodyHandler); + int code = response.statusCode(); + if (code < 200 || code > 299) { + throw new IllegalStateException("Bad response: " + code + " >> " + response.body()); + } + consumer.accept(response, response.body()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + public User user() { + AtomicReference ref = new AtomicReference<>(); + handle( + "/api/user", + HttpRequest.Builder::GET, + HttpResponse.BodyHandlers.ofString(), + (r, b) -> { + try { + User user = MAPPER.readValue(b, User.class); + ref.set(user); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + }); + return ref.get(); + } + + public QueryResult query(String query) { + AtomicReference ref = new AtomicReference<>(); + handle( + "/api/datasources/proxy/1/api/v1/query?query=" + query, + HttpRequest.Builder::GET, + HttpResponse.BodyHandlers.ofString(), + (r, b) -> { + try { + QueryResult result = MAPPER.readValue(b, QueryResult.class); + ref.set(result); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + }); + return ref.get(); + } +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/QuarkusTestResourceTestProfile.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/QuarkusTestResourceTestProfile.java new file mode 100644 index 0000000000000..b60772d61d550 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/QuarkusTestResourceTestProfile.java @@ -0,0 +1,14 @@ +package io.quarkus.observability.test.support; + +import java.util.Map; + +import io.quarkus.test.junit.QuarkusTestProfile; + +public class QuarkusTestResourceTestProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.observability.dev-resources", "false", + "quarkus.observability.enabled", "false"); + } +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/QueryResult.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/QueryResult.java new file mode 100644 index 0000000000000..77f24fa7e5a26 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/QueryResult.java @@ -0,0 +1,73 @@ +package io.quarkus.observability.test.support; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class QueryResult { + public String status; + public Data data; + + // getters and setters + + @Override + public String toString() { + return "QueryResult{" + + "status='" + status + '\'' + + ", data=" + data + + '}'; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Data { + public String resultType; + public List result; + + // getters and setters + + @Override + public String toString() { + return "Data{" + + "resultType='" + resultType + '\'' + + ", result=" + result + + '}'; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ResultItem { + public Metric metric; + public List value; + + // getters and setters + + @Override + public String toString() { + return "ResultItem{" + + "metric=" + metric + + ", value=" + value + + '}'; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Metric { + @JsonProperty("__name__") + public String name; + public String job; + public String test; + + // getters and setters + + @Override + public String toString() { + return "Metric{" + + "name='" + name + '\'' + + ", job='" + job + '\'' + + ", test='" + test + '\'' + + '}'; + } + } +} diff --git a/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/User.java b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/User.java new file mode 100644 index 0000000000000..f617cd2b23bcf --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/java/io/quarkus/observability/test/support/User.java @@ -0,0 +1,14 @@ +package io.quarkus.observability.test.support; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class User { + @JsonProperty + public int id; + @JsonProperty + public String email; + @JsonProperty + public String login; +} diff --git a/integration-tests/observability-lgtm/src/test/resources/application.properties b/integration-tests/observability-lgtm/src/test/resources/application.properties new file mode 100644 index 0000000000000..d3f0cfca1f442 --- /dev/null +++ b/integration-tests/observability-lgtm/src/test/resources/application.properties @@ -0,0 +1,16 @@ +# Disable default binders +quarkus.micrometer.binder-enabled-default=false + +quarkus.log.category."io.quarkus.observability".level=DEBUG +quarkus.log.category."io.quarkus.devservices".level=DEBUG + +#micrometer +quarkus.micrometer.export.otlp.enabled=true +quarkus.micrometer.export.otlp.publish=true +quarkus.micrometer.export.otlp.step=PT5S +quarkus.micrometer.export.otlp.default-registry=true +quarkus.micrometer.export.otlp.url=http://${quarkus.otel-collector.url}/v1/metrics + +#opentelemetry +quarkus.otel.exporter.otlp.traces.protocol=http/protobuf +quarkus.otel.exporter.otlp.traces.endpoint=http://${quarkus.otel-collector.url} diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index a17b168d5cd73..879e46b8c534d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -245,6 +245,7 @@ flyway liquibase liquibase-mongodb + observability-lgtm oidc oidc-client oidc-client-reactive