Make exponential_histogram the default histogram type for HTTP OTLP endpoint

docs/changelog/145065.yaml

@@ -0,0 +1,15 @@
+pr: 145065
+summary: "OTLP endpoint: use exponential_histograms by default"
+area: Mapping
+type: breaking
+breaking:
+  title: Use exponential_histograms by default
+  area: Ingest
+  details: The default of the `xpack.otel_data.histogram_field_type` cluster setting
+    has changed from `histogram` to `exponential_histogram`. This means that histogram
+    fields ingested through the OTLP HTTP endpoint will now be mapped as `exponential_histogram` by default.
+    To maintain the previous behavior, set `xpack.otel_data.histogram_field_type` to `histogram`.
+  impact: Both queryDSL and ES|QL queries will still work as before, but the underlying data structure for histograms will change.
+    The raw field structure will differ, so consumers of raw documents may need to adjust their processing logic if they rely on the specific structure of histogram fields.
+  notable: false
+issues: []

docs/reference/elasticsearch/configuration-reference/index-management-settings.md

@@ -86,15 +86,16 @@
 `xpack.otel_data.registry.enabled`
 :   Specifies whether OpenTelemetry related index templates should be created on startup. Defaults to *true*.
 
-`xpack.otel_data.histogram_field_type` {applies_to}`stack: preview 9.3`
-:   Defines how OTLP histograms are mapped in Elasticsearch. Valid values are:
+`xpack.otel_data.histogram_field_type` {applies_to}`stack: preview 9.3, ga 9.4+` {applies_to}`serverless: ga`
+:   Defines how OTLP histograms are mapped in Elasticsearch.
+    Valid values are:
 
-* `histogram` (default): Map histograms as T-Digests using the `histogram` field type
-* `exponential_histogram`: Map histograms as exponential histograms using the `exponential_histogram` field type
+* `histogram`: Map histograms as T-Digests using the `histogram` field type (Default on {applies_to}`stack: preview =9.3`)
+* `exponential_histogram`: Map histograms as exponential histograms using the `exponential_histogram` field type (Default on {applies_to}`stack: ga 9.4+` {applies_to}`serverless: ga`)
 
 ## Reindex settings [reindex-settings]
 
 $$$reindex-remote-whitelist$$$
 
 `reindex.remote.whitelist` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
 :   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Specifies the hosts that can be [reindexed from remotely](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-reindex). Expects a YAML array of `host:port` strings. Consists of a comma-delimited list of `host:port` entries. Defaults to `["\*.io:*", "\*.com:*"]`.

x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsIndexingRestIT.java

@@ -230,6 +230,8 @@ public void testCounterMonotonicity() throws Exception {
     }
 
     public void testExponentialHistogramsAsTDigest() throws Exception {
+        setHistogramFieldTypeClusterSetting("histogram");
+
         long now = Clock.getDefault().now();
         export(List.of(createExponentialHistogram(now, "exponential_histogram", DELTA, Attributes.empty())));
 
@@ -246,8 +248,6 @@ public void testExponentialHistogramsAsTDigest() throws Exception {
     }
 
     public void testExponentialHistogramsAsExponentialHistogram() throws Exception {
-        setHistogramFieldTypeClusterSetting("exponential_histogram");
-
         long now = Clock.getDefault().now();
         export(List.of(createExponentialHistogram(now, "exponential_histogram", DELTA, Attributes.empty())));
 
@@ -298,6 +298,8 @@ public void testExponentialHistogramsAsAggregateMetricDouble() throws Exception
     }
 
     public void testHistogramAsTDigest() throws Exception {
+        setHistogramFieldTypeClusterSetting("histogram");
+
         long now = Clock.getDefault().now();
         export(List.of(createHistogram(now, "histogram", DELTA, Attributes.empty())));
 
@@ -315,8 +317,6 @@ public void testHistogramAsTDigest() throws Exception {
     }
 
     public void testHistogramsAsExponentialHistogram() throws Exception {
-        setHistogramFieldTypeClusterSetting("exponential_histogram");
-
         long now = Clock.getDefault().now();
         export(List.of(createHistogram(now, "histogram", DELTA, Attributes.empty())));
 

x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java

@@ -46,10 +46,10 @@ public enum HistogramMappingSettingValues {
         EXPONENTIAL_HISTOGRAM
     };
 
-    public static final Setting<HistogramMappingSettingValues> USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE = Setting.enumSetting(
+    public static final Setting<HistogramMappingSettingValues> HISTOGRAM_FIELD_TYPE_SETTING = Setting.enumSetting(
         HistogramMappingSettingValues.class,
         "xpack.otel_data.histogram_field_type",
-        HistogramMappingSettingValues.HISTOGRAM,
+        HistogramMappingSettingValues.EXPONENTIAL_HISTOGRAM,
         Setting.Property.NodeScope,
         Setting.Property.Dynamic
     );
@@ -100,7 +100,7 @@ public void close() {
 
     @Override
     public List<Setting<?>> getSettings() {
-        return List.of(OTEL_DATA_REGISTRY_ENABLED, USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE);
+        return List.of(OTEL_DATA_REGISTRY_ENABLED, HISTOGRAM_FIELD_TYPE_SETTING);
     }
 
     @Override

x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java

@@ -71,8 +71,8 @@ public OTLPMetricsTransportAction(
     ) {
         super(NAME, transportService, actionFilters, threadPool, client);
         ClusterSettings clusterSettings = clusterService.getClusterSettings();
-        defaultMappingHints = MappingHints.fromSettings(clusterSettings.get(OTelPlugin.USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE));
-        clusterSettings.addSettingsUpdateConsumer(OTelPlugin.USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE, histogramFieldTypeSetting -> {
+        defaultMappingHints = MappingHints.fromSettings(clusterSettings.get(OTelPlugin.HISTOGRAM_FIELD_TYPE_SETTING));
+        clusterSettings.addSettingsUpdateConsumer(OTelPlugin.HISTOGRAM_FIELD_TYPE_SETTING, histogramFieldTypeSetting -> {
             defaultMappingHints = MappingHints.fromSettings(histogramFieldTypeSetting);
         });
         this.clusterService = clusterService;

x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportActionTests.java

@@ -45,7 +45,7 @@ public class OTLPMetricsTransportActionTests extends AbstractOTLPTransportAction
     @Override
     protected AbstractOTLPTransportAction createAction() {
         ClusterService clusterService = mock(ClusterService.class);
-        clusterSettings = new ClusterSettings(Settings.EMPTY, Set.of(OTelPlugin.USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE));
+        clusterSettings = new ClusterSettings(Settings.EMPTY, Set.of(OTelPlugin.HISTOGRAM_FIELD_TYPE_SETTING));
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         ProjectMetadata projectMetadata = ProjectMetadata.builder(ProjectId.DEFAULT).build();
         ClusterState clusterState = ClusterState.builder(new ClusterName("test"))
@@ -95,18 +95,16 @@ protected String dataStreamType() {
     // --- metrics-specific tests ---
 
     public void testMappingHintsSettingsUpdate() throws Exception {
+        assertThat(metricsAction.defaultMappingHints, equalTo(MappingHints.DEFAULT_EXPONENTIAL_HISTOGRAM));
+        assertThat(OTelPlugin.HISTOGRAM_FIELD_TYPE_SETTING.isDynamic(), equalTo(true));
+
+        clusterSettings.applySettings(Settings.builder().put(OTelPlugin.HISTOGRAM_FIELD_TYPE_SETTING.getKey(), "histogram").build());
         assertThat(metricsAction.defaultMappingHints, equalTo(MappingHints.DEFAULT_TDIGEST));
-        assertThat(OTelPlugin.USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE.isDynamic(), equalTo(true));
 
         clusterSettings.applySettings(
-            Settings.builder().put(OTelPlugin.USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE.getKey(), "exponential_histogram").build()
+            Settings.builder().put(OTelPlugin.HISTOGRAM_FIELD_TYPE_SETTING.getKey(), "exponential_histogram").build()
         );
         assertThat(metricsAction.defaultMappingHints, equalTo(MappingHints.DEFAULT_EXPONENTIAL_HISTOGRAM));
-
-        clusterSettings.applySettings(
-            Settings.builder().put(OTelPlugin.USE_EXPONENTIAL_HISTOGRAM_FIELD_TYPE.getKey(), "histogram").build()
-        );
-        assertThat(metricsAction.defaultMappingHints, equalTo(MappingHints.DEFAULT_TDIGEST));
     }
 
     // --- helpers ---