Emit JMX metrics using OTEL SDK instead of APM Agent

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
 
 import java.io.IOException;
@@ -38,6 +39,8 @@
  * server URL and secret key can only be provided when Elasticsearch starts.
  */
 class APMJvmOptions {
+    private static final String OTEL_METRICS_ENABLED_SYSTEM_PROPERTY = "telemetry.otel.metrics.enabled";
+
     /**
      * Contains agent configuration that must always be applied, and cannot be overridden.
      */
@@ -137,14 +140,24 @@ class APMJvmOptions {
      */
     static List<String> apmJvmOptions(Settings settings, @Nullable SecureSettings secrets, Path logsDir, Path tmpdir) throws UserException,
         IOException {
-        final Path agentJar = findAgentJar();
+        boolean tracingEnabled = settings.getAsBoolean("telemetry.tracing.enabled", false);
+        boolean metricsEnabled = settings.getAsBoolean("telemetry.metrics.enabled", false);
+        boolean agentMetricsEnabled = Booleans.parseBoolean(System.getProperty(OTEL_METRICS_ENABLED_SYSTEM_PROPERTY, "false")) == false;
+        boolean attachAgent = tracingEnabled || (metricsEnabled && agentMetricsEnabled);
+
+        final Path agentJar = findAgentJar(System.getProperty("user.dir"));
 
-        if (agentJar == null) {
+        if (attachAgent == false || agentJar == null) {
             return List.of();
         }
 
         final Map<String, String> propertiesMap = extractApmSettings(settings);
 
+        if (metricsEnabled == false || agentMetricsEnabled == false) {
+            propertiesMap.put("metrics_interval", "0s");
+            propertiesMap.put("disable_metrics", "*");
+        }
+
         // Configures a log file to write to. Don't disable writing to a log file,
         // as the agent will then require extra Security Manager permissions when
         // it tries to do something else, and it's just painful.

gradle/verification-metadata.xml

@@ -2244,6 +2244,31 @@
             <sha256 value="4a8f41b93eec51e85fa6b48e43de6785b742316fdd9c9baf595adbce6d5de6af" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opentelemetry.instrumentation" name="opentelemetry-instrumentation-api" version="2.23.0">
+         <artifact name="opentelemetry-instrumentation-api-2.23.0.jar">
+            <sha256 value="c6b86acc7ccb2a6a0979ae73ae686ea4ac8a24e656912bc169216bcfd87f44b1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry.instrumentation" name="opentelemetry-instrumentation-api-incubator" version="2.23.0-alpha">
+         <artifact name="opentelemetry-instrumentation-api-incubator-2.23.0-alpha.jar">
+            <sha256 value="4f068fad9ea65cb1f2213ca50ba154878e7e12c87291f2f2182137e26ab25c31" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry.instrumentation" name="opentelemetry-runtime-telemetry-java17" version="2.23.0-alpha">
+         <artifact name="opentelemetry-runtime-telemetry-java17-2.23.0-alpha.jar">
+            <sha256 value="4489144d20e9ea67405e340f399b840db6a89dcbf1ecd85ee7a8beef81629667" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry.instrumentation" name="opentelemetry-runtime-telemetry-java8" version="2.23.0-alpha">
+         <artifact name="opentelemetry-runtime-telemetry-java8-2.23.0-alpha.jar">
+            <sha256 value="71df1c056b9dca5c41077c70d4028a0f4b45c2b089509727fcfd8672afb5504e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opentelemetry.semconv" name="opentelemetry-semconv" version="1.37.0">
+         <artifact name="opentelemetry-semconv-1.37.0.jar">
+            <sha256 value="693ad6f04f29b4b593a04adef5f575d28b3a91ea3449ab5b1e1e2e5c6efc6cdc" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.ous" name="jtoml" version="2.0.0">
          <artifact name="jtoml-2.0.0.jar">
             <sha256 value="3cabdae2244c999addebb8c31ae452fbdc874b4f26a163539954b8eeb5d6acc6" origin="Generated by Gradle"/>

modules/apm/build.gradle

@@ -14,19 +14,66 @@ esplugin {
 }
 
 def otelVersion = '1.57.0'
-def otelSemconvVersion = '1.21.0-alpha'
+def otelInstrumentationVersion = '2.23.0-alpha'
 
 dependencies {
   implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-context:${otelVersion}"
-  implementation "io.opentelemetry:opentelemetry-semconv:${otelSemconvVersion}"
-  runtimeOnly    "io.opentelemetry:opentelemetry-common:${otelVersion}"
-  runtimeOnly    "co.elastic.apm:elastic-apm-agent-java8:1.55.0"
+  implementation "io.opentelemetry:opentelemetry-sdk:${otelVersion}"
+  implementation "io.opentelemetry:opentelemetry-sdk-metrics:${otelVersion}"
+  implementation("io.opentelemetry:opentelemetry-exporter-otlp:${otelVersion}") {
+    exclude group: 'io.opentelemetry', module: 'opentelemetry-exporter-sender-okhttp'
+  }
+  implementation "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17:${otelInstrumentationVersion}"
+  implementation "io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:${otelInstrumentationVersion}"
+
+  implementation "io.opentelemetry:opentelemetry-sdk-common:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-common:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-sdk-trace:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-sdk-logs:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-exporter-common:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-exporter-otlp-common:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-exporter-sender-jdk:${otelVersion}"
+  runtimeOnly "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:${otelVersion}"
+  runtimeOnly "co.elastic.apm:elastic-apm-agent-java8:1.55.0"
+
+  testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}"
 
   javaRestTestImplementation project(':modules:apm')
   javaRestTestImplementation project(':test:framework')
 }
 
+tasks.named("thirdPartyAudit").configure {
+  ignoreMissingClasses(
+    'com.fasterxml.jackson.core.JsonFactory',
+    'com.fasterxml.jackson.core.JsonGenerator',
+    'com.google.common.io.ByteStreams',
+    'com.google.common.util.concurrent.ListenableFuture',
+    'io.grpc.CallOptions',
+    'io.grpc.Channel',
+    'io.grpc.Drainable',
+    'io.grpc.KnownLength',
+    'io.grpc.ManagedChannel',
+    'io.grpc.MethodDescriptor',
+    'io.grpc.MethodDescriptor$Builder',
+    'io.grpc.MethodDescriptor$Marshaller',
+    'io.grpc.MethodDescriptor$MethodType',
+    'io.grpc.stub.AbstractFutureStub',
+    'io.grpc.stub.AbstractStub',
+    'io.grpc.stub.ClientCalls'
+  )
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'io.opentelemetry.internal.shaded.jctools.util.UnsafeRefArrayAccess',
+    'io.opentelemetry.internal.shaded.jctools.util.UnsafeAccess',
+    'io.opentelemetry.exporter.internal.marshal.UnsafeAccess',
+    'io.opentelemetry.exporter.internal.marshal.UnsafeAccess$UnsafeHolder',
+    'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerLimitField',
+    'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueProducerIndexField',
+    'io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueueConsumerIndexField'
+  )
+}
+
 tasks.named("dependencyLicenses").configure {
   mapping from: /opentelemetry-.*/, to: 'opentelemetry'
 }

modules/apm/src/main/java/module-info.java

@@ -13,8 +13,13 @@
     requires org.elasticsearch.xcontent;
     requires org.apache.logging.log4j;
     requires org.apache.lucene.core;
-    requires io.opentelemetry.context;
     requires io.opentelemetry.api;
+    requires io.opentelemetry.context;
+    requires io.opentelemetry.sdk;
+    requires io.opentelemetry.sdk.metrics;
+    requires io.opentelemetry.exporter.otlp;
+    requires io.opentelemetry.instrumentation.runtime_telemetry_java17;
+    requires io.opentelemetry.sdk.common;
 
     exports org.elasticsearch.telemetry.apm;
     exports org.elasticsearch.telemetry.apm.metrics;

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.telemetry.apm.internal.APMAgentSettings;
 import org.elasticsearch.telemetry.apm.internal.APMMeterService;
 import org.elasticsearch.telemetry.apm.internal.APMTelemetryProvider;
+import org.elasticsearch.telemetry.apm.internal.OTelSdkSettings;
 import org.elasticsearch.telemetry.apm.internal.tracing.APMTracer;
 
 import java.util.Collection;
@@ -88,6 +89,8 @@ public List<Setting<?>> getSettings() {
             APMAgentSettings.TELEMETRY_API_KEY_SETTING,
             // Metrics
             APMAgentSettings.TELEMETRY_METRICS_ENABLED_SETTING,
+            OTelSdkSettings.TELEMETRY_OTEL_METRICS_ENDPOINT,
+            OTelSdkSettings.TELEMETRY_OTEL_METRICS_INTERVAL,
             // Tracing
             APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING,
             APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING,

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMMeterService.java

@@ -16,20 +16,22 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.telemetry.apm.APMMeterRegistry;
 
 import java.util.function.Supplier;
 
 public class APMMeterService extends AbstractLifecycleComponent {
-    private final APMMeterRegistry meterRegistry;
+    static final String OTEL_METRICS_ENABLED_SYSTEM_PROPERTY = "telemetry.otel.metrics.enabled";
 
+    private final APMMeterRegistry meterRegistry;
     private final Supplier<Meter> otelMeterSupplier;
     private final Supplier<Meter> noopMeterSupplier;
 
     protected volatile boolean enabled;
 
     public APMMeterService(Settings settings) {
-        this(settings, APMMeterService.otelMeter(), APMMeterService.noopMeter());
+        this(settings, createOtelMeterSupplier(settings), () -> OpenTelemetry.noop().getMeter("noop"));
     }
 
     public APMMeterService(Settings settings, Supplier<Meter> otelMeterSupplier, Supplier<Meter> noopMeterSupplier) {
@@ -40,7 +42,14 @@ public APMMeterService(boolean enabled, Supplier<Meter> otelMeterSupplier, Suppl
         this.enabled = enabled;
         this.otelMeterSupplier = otelMeterSupplier;
         this.noopMeterSupplier = noopMeterSupplier;
-        this.meterRegistry = new APMMeterRegistry(enabled ? createOtelMeter() : createNoopMeter());
+        this.meterRegistry = new APMMeterRegistry(enabled ? otelMeterSupplier.get() : noopMeterSupplier.get());
+    }
+
+    private static Supplier<Meter> createOtelMeterSupplier(Settings settings) {
+        if (Booleans.parseBoolean(System.getProperty(OTEL_METRICS_ENABLED_SYSTEM_PROPERTY, "false")) == false) {
+            return () -> GlobalOpenTelemetry.get().getMeter("elasticsearch");
+        }
+        return new OTelSdkMeterSupplier(settings);
     }
 
     public APMMeterRegistry getMeterRegistry() {
@@ -52,41 +61,24 @@ public APMMeterRegistry getMeterRegistry() {
      */
     void setEnabled(boolean enabled) {
         this.enabled = enabled;
-        if (enabled) {
-            meterRegistry.setProvider(createOtelMeter());
-        } else {
-            meterRegistry.setProvider(createNoopMeter());
-        }
+        meterRegistry.setProvider(enabled ? otelMeterSupplier.get() : noopMeterSupplier.get());
     }
 
     @Override
     protected void doStart() {}
 
     @Override
     protected void doStop() {
-        meterRegistry.setProvider(createNoopMeter());
+        if (otelMeterSupplier instanceof AutoCloseable closeable) {
+            try {
+                closeable.close();
+            } catch (Exception e) {
+                // TODO
+            }
+        }
+        meterRegistry.setProvider(noopMeterSupplier.get());
     }
 
     @Override
     protected void doClose() {}
-
-    protected Meter createOtelMeter() {
-        assert this.enabled;
-        return otelMeterSupplier.get();
-    }
-
-    protected Meter createNoopMeter() {
-        return noopMeterSupplier.get();
-    }
-
-    protected static Supplier<Meter> noopMeter() {
-        return () -> OpenTelemetry.noop().getMeter("noop");
-    }
-
-    // to be used within doPrivileged block
-    private static Supplier<Meter> otelMeter() {
-        var openTelemetry = GlobalOpenTelemetry.get();
-        var meter = openTelemetry.getMeter("elasticsearch");
-        return () -> meter;
-    }
 }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/MetricValidator.java

@@ -67,7 +67,14 @@ public class MetricValidator {
         "es.thread_pool.searchable_snapshots_cache_fetch_async.*",
         "es.thread_pool.searchable_snapshots_cache_prewarming.*",
         "es.thread_pool.security-crypto.*",
-        "es.thread_pool.security-token-key.*"
+        "es.thread_pool.security-token-key.*",
+        // APM Java agent-compatible metric names (see https://www.elastic.co/docs/reference/apm/agents/java/metrics#metrics-jvm)
+        "system.cpu.*",
+        "system.memory.*",
+        "system.process.*",
+        "jvm.file_descriptor.*",
+        "jvm.gc.*",
+        "jvm.memory.*"
     );
 
     /**
@@ -241,7 +248,10 @@ private static class Attributes {
             Map.entry("es.tsdb.downsample.actions.shard.total", DOWNSAMPLE_ATTRIBUTES),
             Map.entry("es.tsdb.downsample.actions.total", DOWNSAMPLE_ATTRIBUTES),
             Map.entry("es.tsdb.downsample.latency.shard.histogram", DOWNSAMPLE_ATTRIBUTES),
-            Map.entry("es.tsdb.downsample.latency.total.histogram", DOWNSAMPLE_ATTRIBUTES)
+            Map.entry("es.tsdb.downsample.latency.total.histogram", DOWNSAMPLE_ATTRIBUTES),
+            // APM Java agent-compatible metrics (see https://www.elastic.co/docs/reference/apm/agents/java/metrics#metrics-jvm)
+            Map.entry("jvm.gc.count", Set.of("name")),
+            Map.entry("jvm.gc.time", Set.of("name"))
         );
 
         // forbidden attributes known to cause issues due to mapping conflicts or high cardinality
@@ -317,6 +327,11 @@ public static void assertValidAttributeNames(String metricName, Map<String, Obje
 
             assert Attributes.OTEL_ATTRIBUTES.contains(attribute)
                 || Attributes.SKIP_VALIDATION.getOrDefault(metricName, emptySet()).contains(attribute)
+                // allow percentile for all thread pools
+                // https://github.com/elastic/dev/issues/3436 remove the usage of percentile as attribute and move to metric name.
+                || (metricName.startsWith("es.thread_pool.") && attribute.equals("percentile"))
+                // ML metrics use dot-separated attribute key
+                || (metricName.startsWith("es.ml.") && attribute.equals("es.ml.is_master"))
                 || Attributes.ATTRIBUTE_PATTERN.matcher(attribute).matches()
                 : Strings.format(
                     "Attribute [%s] of [%s] does not match the required naming pattern [%s], see the naming guidelines.",