Emit JVM metrics with OTEL SDK

build-tools/build.gradle

@@ -143,6 +143,7 @@ dependencies {
     implementation buildLibs.asm
     implementation buildLibs.jackson.core
     implementation buildLibs.jackson.databind
+    implementation "io.opentelemetry.proto:opentelemetry-proto:1.7.0-alpha"
 
     testFixturesApi buildLibs.commmons.io
     testFixturesApi gradleApi()

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.gradle.testclusters;
 
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -89,6 +91,7 @@ public void start() throws IOException {
         }
         InetSocketAddress addr = new InetSocketAddress("0.0.0.0", 0);
         HttpServer server = HttpServer.create(addr, 10);
+        server.createContext("/v1/metrics", new OtlpMetricsHandler());
         server.createContext("/", new RootHandler());
         server.start();
         instance = server;
@@ -209,4 +212,30 @@ private void logFiltered(InputStream body) throws IOException {
             }
         }
     }
+
+    class OtlpMetricsHandler implements HttpHandler {
+        @Override
+        public void handle(HttpExchange t) throws IOException {
+            byte[] bytes = t.getRequestBody().readAllBytes();
+            ExportMetricsServiceRequest metrics = ExportMetricsServiceRequest.parseFrom(bytes);
+            for (var resourceMetrics : metrics.getResourceMetricsList()) {
+                var samples = new ArrayList<String>();
+                for (var scopeMetrics : resourceMetrics.getScopeMetricsList()) {
+                    for (var metric : scopeMetrics.getMetricsList()) {
+                        String name = metric.getName();
+                        if (metricFilter != null && metricFilter.matcher(name).matches() == false) {
+                            continue;
+                        }
+                        samples.add(metric.toString());
+                    }
+                }
+                if (samples.isEmpty() == false) {
+                    logger.lifecycle("OTLP Metricset:\n{}", String.join("\n", samples));
+                }
+            }
+
+            t.sendResponseHeaders(200, 0);
+            t.getResponseBody().close();
+        }
+    }
 }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -48,6 +48,8 @@ public abstract class RunTask extends DefaultTestClustersTask {
 
     private Boolean apmServerEnabled = false;
 
+    private Boolean usingOtelSdk = false;
+
     private String apmServerMetrics = null;
 
     private String apmServerTransactions = null;
@@ -130,6 +132,19 @@ public void setApmServerEnabled(Boolean apmServerEnabled) {
         this.apmServerEnabled = apmServerEnabled;
     }
 
+    @Input
+    public Boolean getUsingOtelSdk() {
+        return usingOtelSdk;
+    }
+
+    @Option(
+        option = "using-otel-sdk",
+        description = "Use the OTel SDK for metrics export instead of the APM agent (requires --with-apm-server)"
+    )
+    public void setUsingOtelSdk(Boolean usingOtelSdk) {
+        this.usingOtelSdk = usingOtelSdk;
+    }
+
     @Option(option = "apm-metrics", description = "Metric wildcard filter for APM server")
     public void setApmServerMetrics(String apmServerMetrics) {
         this.apmServerMetrics = apmServerMetrics;
@@ -245,6 +260,10 @@ public void beforeStart() {
             getDataPath = n -> dataDir.resolve(n.getName());
         }
 
+        if (usingOtelSdk && apmServerEnabled == false) {
+            throw new GradleException("--using-otel-sdk requires --with-apm-server");
+        }
+
         if (apmServerEnabled) {
             try {
                 mockServer = new MockApmServer(apmServerMetrics, apmServerTransactions, apmServerTransactionsExcludes);
@@ -285,10 +304,16 @@ public void beforeStart() {
                 if (mockServer != null) {
                     node.setting("telemetry.metrics.enabled", "true");
                     node.setting("telemetry.tracing.enabled", "true");
-                    node.setting("telemetry.agent.transaction_sample_rate", "1.0");
-                    node.setting("telemetry.agent.transaction_max_spans", "100");
-                    node.setting("telemetry.agent.metrics_interval", "10s");
                     node.setting("telemetry.agent.server_url", "http://127.0.0.1:" + mockServer.getPort());
+                    if (usingOtelSdk) {
+                        node.systemProperty("telemetry.otel.metrics.enabled", "true");
+                        node.setting("telemetry.otel.metrics.endpoint", "http://127.0.0.1:" + mockServer.getPort() + "/v1/metrics");
+                        node.setting("telemetry.otel.metrics.interval", "10s");
+                    } else {
+                        node.setting("telemetry.agent.transaction_sample_rate", "1.0");
+                        node.setting("telemetry.agent.transaction_max_spans", "100");
+                        node.setting("telemetry.agent.metrics_interval", "10s");
+                    }
                 }
                 // in serverless metrics are enabled by default
                 // if metrics were not enabled explicitly for gradlew run we should disable them

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
 
 import java.io.IOException;
@@ -31,13 +32,16 @@
 import java.util.Set;
 import java.util.StringJoiner;
 
+import static org.elasticsearch.telemetry.TelemetryProvider.OTEL_METRICS_ENABLED_SYSTEM_PROPERTY;
+
 /**
  * This class is responsible for working out if APM telemetry is configured and if so, preparing
  * a temporary config file for the APM Java agent and CLI options to the JVM to configure APM.
  * APM doesn't need to be enabled, as that can be toggled at runtime, but some configuration e.g.
  * server URL and secret key can only be provided when Elasticsearch starts.
  */
 class APMJvmOptions {
+
     /**
      * Contains agent configuration that must always be applied, and cannot be overridden.
      */
@@ -137,14 +141,24 @@ class APMJvmOptions {
      */
     static List<String> apmJvmOptions(Settings settings, @Nullable SecureSettings secrets, Path logsDir, Path tmpdir) throws UserException,
         IOException {
+        boolean tracingEnabled = settings.getAsBoolean("telemetry.tracing.enabled", false);
+        boolean metricsEnabled = settings.getAsBoolean("telemetry.metrics.enabled", false);
+        boolean agentMetricsEnabled = Booleans.parseBoolean(System.getProperty(OTEL_METRICS_ENABLED_SYSTEM_PROPERTY, "false")) == false;
+        boolean attachAgent = tracingEnabled || (metricsEnabled && agentMetricsEnabled);
+
         final Path agentJar = findAgentJar();
 
-        if (agentJar == null) {
+        if (attachAgent == false || agentJar == null) {
             return List.of();
         }
 
         final Map<String, String> propertiesMap = extractApmSettings(settings);
 
+        if (metricsEnabled == false || agentMetricsEnabled == false) {
+            propertiesMap.put("metrics_interval", "0s");
+            propertiesMap.put("disable_metrics", "*");
+        }
+
         // Configures a log file to write to. Don't disable writing to a log file,
         // as the agent will then require extra Security Manager permissions when
         // it tries to do something else, and it's just painful.