Enable failure store for OTEL datastreams

docs/changelog/131395.yaml

@@ -0,0 +1,5 @@
+pr: 131395
+summary: Enable failure store for newly created OTel data streams
+area: Data streams
+type: enhancement
+issues: []

x-pack/plugin/otel-data/src/main/resources/component-templates/otel@settings.yaml

@@ -0,0 +1,8 @@
+version: ${xpack.oteldata.template.version}
+_meta:
+  description: Default settings for all OpenTelemetry data streams
+  managed: true
+template:
+  data_stream_options:
+    failure_store:
+      enabled: true

x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml

@@ -11,6 +11,7 @@ composed_of:
   - logs@mappings
   - logs@settings
   - otel@mappings
+  - otel@settings
   - logs-otel@mappings
   - semconv-resource-to-ecs@mappings
   - logs@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml

@@ -10,6 +10,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml

@@ -10,6 +10,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml

@@ -10,6 +10,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml

@@ -10,6 +10,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml

@@ -10,6 +10,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml

@@ -11,6 +11,7 @@ _meta:
 composed_of:
   - metrics@tsdb-settings
   - otel@mappings
+  - otel@settings
   - metrics-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics@custom

x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml

@@ -11,6 +11,7 @@ composed_of:
   - traces@mappings
   - traces@settings
   - otel@mappings
+  - otel@settings
   - traces-otel@mappings
   - semconv-resource-to-ecs@mappings
   - traces@custom

x-pack/plugin/otel-data/src/main/resources/resources.yaml

@@ -1,10 +1,11 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin otel-data. This must be increased whenever an existing template is
 # changed, in order for it to be updated on Elasticsearch upgrade.
-version: 9
+version: 10
 
 component-templates:
   - otel@mappings
+  - otel@settings
   - logs-otel@mappings
   - semconv-resource-to-ecs@mappings
   - metrics-otel@mappings

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_failure_store_test.yml

@@ -0,0 +1,73 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+---
+teardown:
+  - do:
+      indices.delete_data_stream:
+        name: logs-generic.otel-default
+        ignore: 404
+---
+"Test logs-*.otel-* data streams have failure store enabled by default":
+  # Index a valid document (string message).
+  - do:
+      index:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          '@timestamp': '2023-01-01T12:00:00Z'
+          severity_text: "INFO"
+          text: "Application started successfully"
+  - match: { result: created }
+
+  # Assert empty failure store.
+  - do:
+      indices.get_data_stream:
+        name: logs-generic.otel-default
+  - match: { data_streams.0.name: logs-generic.otel-default }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 0 }
+
+  # Index a document with naming alias, causing an error.
+  - do:
+      index:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          '@timestamp': '2023-01-01T12:01:00Z'
+          severity_text: "ERROR"
+          message: "Application started successfully"
+  - match: { result: 'created' }
+  - match: { failure_store: used}
+
+  # Assert failure store containing 1 item.
+  - do:
+      indices.get_data_stream:
+        name: logs-generic.otel-default
+  - length: { data_streams.0.failure_store.indices: 1 }
+
+  # Assert valid document.
+  - do:
+      search:
+        index: logs-generic.otel-default::data
+        body:
+          query:
+            match_all: {}
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.severity_text: "INFO" }
+  - match: { hits.hits.0._source.text: "Application started successfully" }
+
+  # Assert invalid document.
+  - do:
+      search:
+        index: logs-generic.otel-default::failures
+        body:
+          query:
+            match_all: {}
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.document.source.severity_text: "ERROR" }
+  - match: { hits.hits.0._source.document.source.message: "Application started successfully" }
+  - match: { hits.hits.0._source.error.type: "document_parsing_exception" }