diff --git a/docs/changelog/131395.yaml b/docs/changelog/131395.yaml new file mode 100644 index 0000000000000..500b761be1472 --- /dev/null +++ b/docs/changelog/131395.yaml @@ -0,0 +1,5 @@ +pr: 131395 +summary: Enable failure store for newly created OTel data streams +area: Data streams +type: enhancement +issues: [] diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@settings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@settings.yaml new file mode 100644 index 0000000000000..0d3b1b25a7ea9 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@settings.yaml @@ -0,0 +1,8 @@ +version: ${xpack.oteldata.template.version} +_meta: + description: Default settings for all OpenTelemetry data streams + managed: true +template: + data_stream_options: + failure_store: + enabled: true diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml index 6772ec5bc65d4..929d26e1c30af 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml @@ -11,6 +11,7 @@ composed_of: - logs@mappings - logs@settings - otel@mappings + - otel@settings - logs-otel@mappings - semconv-resource-to-ecs@mappings - logs@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index f8489605ad1bf..a042fc77e6fa3 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -10,6 +10,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml index f5033135120bc..60739559cc9eb 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml index 9168062f30bfb..9464936f5e1e5 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml @@ -10,6 +10,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml index 47c2d7d014322..888a2145073fd 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml index c9438e8c27402..36be8cb78d851 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml index b29caa3fe34a7..20d1e3ca65e88 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml @@ -10,6 +10,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml index 4cab3e41a1dfa..9bb62ae9edd3b 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml index 037f3546205d6..ff4780744e216 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml index 303ac2c406fd0..b1037535754f3 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml @@ -10,6 +10,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml index ea42079ced4dd..15088a2198abc 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml index 81e70cc3361fc..2f6f7e28ffc22 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml index c54b90bf8b683..5cc1828d3285b 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml @@ -10,6 +10,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml index 8afe8b87951c0..906e535e2c05b 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml @@ -11,6 +11,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - otel@settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index 370b9351c16f5..c2e9a68bc72ad 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -11,6 +11,7 @@ composed_of: - traces@mappings - traces@settings - otel@mappings + - otel@settings - traces-otel@mappings - semconv-resource-to-ecs@mappings - traces@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index 6aadfde1683dc..608dc369c34eb 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,10 +1,11 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin otel-data. This must be increased whenever an existing template is # changed, in order for it to be updated on Elasticsearch upgrade. -version: 9 +version: 10 component-templates: - otel@mappings + - otel@settings - logs-otel@mappings - semconv-resource-to-ecs@mappings - metrics-otel@mappings diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_failure_store_test.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_failure_store_test.yml new file mode 100644 index 0000000000000..dfc6d0fc050b0 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_failure_store_test.yml @@ -0,0 +1,73 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid +--- +teardown: + - do: + indices.delete_data_stream: + name: logs-generic.otel-default + ignore: 404 +--- +"Test logs-*.otel-* data streams have failure store enabled by default": + # Index a valid document (string message). + - do: + index: + index: logs-generic.otel-default + refresh: true + body: + '@timestamp': '2023-01-01T12:00:00Z' + severity_text: "INFO" + text: "Application started successfully" + - match: { result: created } + + # Assert empty failure store. + - do: + indices.get_data_stream: + name: logs-generic.otel-default + - match: { data_streams.0.name: logs-generic.otel-default } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.failure_store.enabled: true } + - length: { data_streams.0.failure_store.indices: 0 } + + # Index a document with naming alias, causing an error. + - do: + index: + index: logs-generic.otel-default + refresh: true + body: + '@timestamp': '2023-01-01T12:01:00Z' + severity_text: "ERROR" + message: "Application started successfully" + - match: { result: 'created' } + - match: { failure_store: used} + + # Assert failure store containing 1 item. + - do: + indices.get_data_stream: + name: logs-generic.otel-default + - length: { data_streams.0.failure_store.indices: 1 } + + # Assert valid document. + - do: + search: + index: logs-generic.otel-default::data + body: + query: + match_all: {} + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.severity_text: "INFO" } + - match: { hits.hits.0._source.text: "Application started successfully" } + + # Assert invalid document. + - do: + search: + index: logs-generic.otel-default::failures + body: + query: + match_all: {} + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.document.source.severity_text: "ERROR" } + - match: { hits.hits.0._source.document.source.message: "Application started successfully" } + - match: { hits.hits.0._source.error.type: "document_parsing_exception" }