From 96c5c22f63a0fdb0b2bfd5747f6503fab3e1c03b Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 18 Jul 2024 15:59:34 +0200 Subject: [PATCH 01/51] Add YamlTemplateRegistry and OtelIndexTemplateRegistry with resource YAML files --- .../xpack/core/ClientHelper.java | 1 + .../xpack/core/template/ResourceUtils.java | 39 ++++ .../xpack/core/template/TemplateUtils.java | 5 + .../template/YamlIngestPipelineConfig.java | 30 +++ .../core/template/YamlTemplateRegistry.java | 208 ++++++++++++++++++ x-pack/plugin/otel-data/README.md | 44 ++++ x-pack/plugin/otel-data/build.gradle | 37 ++++ .../xpack/oteldata/OTelPlugin.java | 79 +++++++ .../oteldata/OtelIndexTemplateRegistry.java | 42 ++++ .../ecs-tsdb@mappings.yaml | 20 ++ .../logs-otel@mappings.yaml | 41 ++++ .../metrics-otel@mappings.yaml | 55 +++++ .../component-templates/otel@mappings.yaml | 63 ++++++ .../semconv-resource-to-ecs@mappings.yaml | 119 ++++++++++ .../tracer-otel@mappings.yaml | 71 ++++++ .../component-templates/traces@lifecycle.yaml | 13 ++ .../component-templates/traces@mappings.yaml | 17 ++ .../component-templates/traces@settings.yaml | 12 + .../main/resources/index-templates/README.md | 7 + .../index-templates/logs-otel@template.yaml | 21 ++ .../metrics-otel@template.yaml | 23 ++ .../index-templates/traces-otel@template.yaml | 21 ++ .../src/main/resources/resources.yaml | 20 ++ 23 files changed, 988 insertions(+) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java create mode 100644 x-pack/plugin/otel-data/README.md create mode 100644 x-pack/plugin/otel-data/build.gradle create mode 100644 x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java create mode 100644 x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/tracer-otel@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/README.md create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/resources.yaml diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index d27d7a21ddb73..4e7aa37fe1a0b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -194,6 +194,7 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion( public static final String CONNECTORS_ORIGIN = "connectors"; public static final String INFERENCE_ORIGIN = "inference"; public static final String APM_ORIGIN = "apm"; + public static final String OTEL_ORIGIN = "otel"; private ClientHelper() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java new file mode 100644 index 0000000000000..2ca3c5ec4fa02 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class ResourceUtils +{ + static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty) { + return loadVersionedResourceUTF8(clazz, name, version, versionProperty, Map.of()); + } + + static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty, Map variables) { + try { + String content = loadResource(clazz, name); + content = TemplateUtils.replaceVariables(content, String.valueOf(version), versionProperty, variables); + return content.getBytes(StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static String loadResource(Class clazz, String name) throws IOException { + InputStream is = clazz.getResourceAsStream(name); + if (is == null) { + throw new IOException("Resource [" + name + "] not found in classpath."); + } + return new String(is.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java index ba9639d3d5156..f9794000ee295 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java @@ -10,10 +10,15 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.template.resources.TemplateResources; +import org.elasticsearch.xpack.core.watcher.input.Input; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java new file mode 100644 index 0000000000000..d88b17bed541a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentType; +import java.util.List; + +public class YamlIngestPipelineConfig extends IngestPipelineConfig { + public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies) { + super(id, resource, version, versionProperty, dependencies); + } + + @Override + public XContentType getXContentType() { + return XContentType.YAML; + } + + @Override + public BytesReference loadConfig() { + // return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version, variables)); + return null; // TODO + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java new file mode 100644 index 0000000000000..7f528bf9d05b1 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.yaml.YamlXContent; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource; +import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; + +/** + * Creates index templates and ingest pipelines based on YAML files from resources. + */ +public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { + private static final Logger logger = LogManager.getLogger(YamlTemplateRegistry.class); + // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a + // dependency to the data-streams module just for this + public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); + private final int version; + + private final Map componentTemplates; + private final Map composableIndexTemplates; + private final List ingestPipelines; + private final FeatureService featureService; + private volatile boolean enabled; + + @SuppressWarnings("unchecked") + public YamlTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + FeatureService featureService + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + + try { + + var clazz = this.getClass(); + logger.info("========= classname: " + clazz.getName()); + + final Map resources = XContentHelper.convertToMap( + YamlXContent.yamlXContent, + loadResource(this.getClass(), "/resources.yaml"), + false + ); + version = (((Number) resources.get("version")).intValue()); + logger.info("========= version: " + version); + + final List componentTemplateNames = (List) resources.get("component-templates"); + final List indexTemplateNames = (List) resources.get("index-templates"); + final List ingestPipelineConfigs = (List) resources.get("ingest-pipelines"); + + componentTemplates = Optional.ofNullable(componentTemplateNames) + .orElse(Collections.emptyList()) + .stream() + .map(o -> (String) o) + .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); + composableIndexTemplates = Optional.ofNullable(indexTemplateNames) + .orElse(Collections.emptyList()) + .stream() + .map(o -> (String) o) + .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); + ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) + .orElse(Collections.emptyList()) + .stream().map(o -> (Map>) o).map(map -> { + Map.Entry> pipelineConfig = map.entrySet().iterator().next(); + return loadIngestPipeline(pipelineConfig.getKey(), version, (List) pipelineConfig.getValue().get("dependencies")); + }).collect(Collectors.toList()); + this.featureService = featureService; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public int getVersion() { + return version; + } + + /*** + * + * @return A friendly, human readable name of the index template regisry + */ + public abstract String getName(); + + public void setEnabled(boolean enabled) { + logger.info("{} index template registry is {}", getName(), enabled ? "enabled" : "disabled"); + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } + + public void close() { + clusterService.removeListener(this); + } + + @Override + protected boolean isClusterReady(ClusterChangedEvent event) { + // Ensure current version of the components are installed only after versions that support data stream lifecycle + // due to the use of the feature in all the `@lifecycle` component templates + return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE); + } + + @Override + protected boolean requiresMasterNode() { + return true; + } + + @Override + protected Map getComponentTemplateConfigs() { + if (enabled) { + return componentTemplates; + } else { + return Map.of(); + } + } + + @Override + protected Map getComposableTemplateConfigs() { + if (enabled) { + return composableIndexTemplates; + } else { + return Map.of(); + } + } + + @Override + protected List getIngestPipelines() { + if (enabled) { + return ingestPipelines; + } else { + return Collections.emptyList(); + } + } + + protected abstract String getVersionProperty(); + + private ComponentTemplate loadComponentTemplate(String name, int version) { + try { + final byte[] content = loadVersionedResourceUTF8(this.getClass(), "/component-templates/" + + name + ".yaml", version, getVersionProperty()); + try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { + return ComponentTemplate.parse(parser); + } + } catch (Exception e) { + throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); + } + } + + private ComposableIndexTemplate loadIndexTemplate(String name, int version) { + try { + final byte[] content = loadVersionedResourceUTF8(this.getClass(), "/index-templates/" + + name + ".yaml", version, getVersionProperty()); + try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { + return ComposableIndexTemplate.parse(parser); + } + } catch (Exception e) { + throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); + } + } + + private static IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { + if (dependencies == null) { + dependencies = Collections.emptyList(); + } + return new YamlIngestPipelineConfig( + name, + "/ingest-pipelines/" + name + ".yaml", + version, + "TODO", //APM_TEMPLATE_VERSION_VARIABLE //TODO + dependencies + ); + } + + @Override + protected boolean applyRolloverAfterTemplateV2Upgrade() { + return true; + } +} diff --git a/x-pack/plugin/otel-data/README.md b/x-pack/plugin/otel-data/README.md new file mode 100644 index 0000000000000..862ec7cbb025b --- /dev/null +++ b/x-pack/plugin/otel-data/README.md @@ -0,0 +1,44 @@ +NOTE: this plugin is not related to APM Metrics used in ES codebase. The APM Metrics are in :modules:apm + +## OpenTelemetry Ingest plugin + +The OpenTelemetry Ingest plugin installs index templates and component templates for Elastic OpenTelemetry data. + +All resources are defined as YAML under [src/main/resources](src/main/resources). + +The OpenTelemetry index templates rely on mappings from `x-pack-core`. +See [x-pack/plugin/core/src/main/resources](../core/src/main/resources). + +## Adding/Removing/Updating a resource + +All resources are defined as YAML under [src/main/resources](src/main/resources). + +For a resource to be known to the plugin it must be added to +[src/main/resources/resources.yaml](src/main/resources/resources.yaml) in the +appropriate section. + +Any update to resources included by this package also requires a bump to the +`version` property included in the resources file. + +## Testing + +## Unit testing + +Java unit tests cover basic, low-level details of the plugin, such as the parsing and loading of resources. +These can be run with: + +``` +./gradlew x-pack:plugin:apm-data:test +``` + +## Integration testing + +The index templates and ingest pipeline functionality is tested using YAML REST tests. +These can be run with: + +``` +./gradlew x-pack:plugin:apm-data:yamlRestTest +``` + +Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc) +for information about writing YAML REST tests. diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle new file mode 100644 index 0000000000000..fe030adee26dc --- /dev/null +++ b/x-pack/plugin/otel-data/build.gradle @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + name 'x-pack-otel-data' + description 'The OTEL plugin defines OTEL data streams and ingest pipelines.' + classname 'org.elasticsearch.xpack.oteldata.OTelPlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation project(path: ':x-pack:plugin:stack') + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(':modules:data-streams') + clusterModules project(':modules:data-streams') + clusterModules project(':modules:ingest-common') + clusterModules project(':modules:ingest-geoip') + clusterModules project(':modules:ingest-user-agent') + clusterModules project(':modules:lang-mustache') + clusterModules project(':modules:mapper-extras') + clusterModules project(xpackModule('analytics')) + clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-aggregate-metric')) + clusterModules project(xpackModule('mapper-constant-keyword')) + clusterModules project(xpackModule('mapper-counted-keyword')) + clusterModules project(xpackModule('stack')) + clusterModules project(xpackModule('wildcard')) +} diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java new file mode 100644 index 0000000000000..905eceecc7fa8 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.oteldata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackSettings; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + + +public class OTelPlugin extends Plugin implements ActionPlugin { + private static final Logger logger = LogManager.getLogger(OTelPlugin.class); + + final SetOnce registry = new SetOnce<>(); + + private final boolean enabled; + + // OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry. + // + // This setting will be ignored if the plugin is disabled. + static final Setting OTEL_DATA_REGISTRY_ENABLED = Setting.boolSetting( + "xpack.otel_data.registry.enabled", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public OTelPlugin(Settings settings) { + this.enabled = XPackSettings.APM_DATA_ENABLED.get(settings); + } + + @Override + public Collection createComponents(PluginServices services) { + logger.info("APM ingest plugin is {}", enabled ? "enabled" : "disabled"); + Settings settings = services.environment().settings(); + ClusterService clusterService = services.clusterService(); + registry.set( + new OtelIndexTemplateRegistry( + settings, + clusterService, + services.threadPool(), + services.client(), + services.xContentRegistry(), + services.featureService() + ) + ); + if (enabled) { + OtelIndexTemplateRegistry registryInstance = registry.get(); + registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings)); + clusterService.getClusterSettings().addSettingsUpdateConsumer(OTEL_DATA_REGISTRY_ENABLED, registryInstance::setEnabled); + registryInstance.initialize(); + } + return Collections.emptyList(); + } + + @Override + public void close() { + registry.get().close(); + } + + @Override + public List> getSettings() { + return List.of(OTEL_DATA_REGISTRY_ENABLED); + } +} diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java new file mode 100644 index 0000000000000..83f0164d5ca60 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.oteldata; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; + +public class OtelIndexTemplateRegistry extends YamlTemplateRegistry { + + public static final String OTEL_TEMPLATE_VERSION_VARIABLE = "xpack.oteldata.template.version"; + + public OtelIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, + NamedXContentRegistry xContentRegistry, FeatureService featureService) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); + } + + @Override + protected String getOrigin() { + return ClientHelper.OTEL_ORIGIN; + } + + @Override + public String getName() { + return "OpenTelemetry"; + } + + @Override + protected String getVersionProperty() { + return OTEL_TEMPLATE_VERSION_VARIABLE; + } +} diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml new file mode 100644 index 0000000000000..eed2b0c605175 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -0,0 +1,20 @@ +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for the logs index template installed by x-pack + managed: true +template: + mappings: + dynamic_templates: + - ecs_ip: + mapping: + type: ip + path_match: + - "ip" + - "*.ip" + - "*_ip" + match_mapping_type: string + - all_strings_to_keywords: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml new file mode 100644 index 0000000000000..ad505029ea9bf --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -0,0 +1,41 @@ +--- +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for the logs index template installed by x-pack + managed: true +template: + mappings: + date_detection: false + dynamic: strict + properties: + data_stream.type: + type: constant_keyword + value: logs + observed_timestamp: + type: date_nanos + severity_number: + type: byte + severity_text: + type: keyword + log.level: + type: alias + path: severity_text + body_text: + type: match_only_text + message: + type: alias + path: body_text + body_structured: + type: flattened + trace_flags: + type: byte + trace_id: + type: keyword + trace.id: + type: alias + path: trace_id + span_id: + type: keyword + span.id: + type: alias + path: span_id diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml new file mode 100644 index 0000000000000..b8751ef60aca6 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -0,0 +1,55 @@ +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for the logs index template installed by x-pack + managed: true +template: + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + start_time: + type: date_nanos + metrics: + type: passthrough + dynamic: true + priority: 1 + unit: + type: keyword + time_series_dimension: true + ignore_above: 1024 + dynamic_templates: + - histogram: + mapping: + type: histogram + ignore_malformed: true + - counter_long: + mapping: + type: long + time_series_metric: counter + ignore_malformed: true + - gauge_long: + mapping: + type: long + time_series_metric: gauge + ignore_malformed: true + - counter_double: + mapping: + type: double + time_series_metric: counter + ignore_malformed: true + - gauge_double: + mapping: + type: double + time_series_metric: gauge + ignore_malformed: true + - summary_gauge: + mapping: + type: aggregate_metric_double + time_series_metric: gauge + ignore_malformed: true + - summary_counter: + mapping: + type: aggregate_metric_double + time_series_metric: counter + ignore_malformed: true diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml new file mode 100644 index 0000000000000..0a6e571f415f5 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -0,0 +1,63 @@ +--- +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for all OTel data streams + managed: true +template: + mappings: + date_detection: false + dynamic: strict + properties: + "@timestamp": + type: date_nanos + data_stream.type: + type: constant_keyword + data_stream.dataset: + type: constant_keyword + data_stream.namespace: + type: constant_keyword + attributes: + type: passthrough + dynamic: true + priority: 10 + time_series_dimension: true + dropped_attributes_count: + type: long + scope: + properties: + name: + type: keyword + ignore_above: 1024 + version: + type: version + schema_url: + type: keyword + ignore_above: 1024 + dropped_attributes_count: + type: long + attributes: + type: passthrough + dynamic: true + priority: 20 + time_series_dimension: true + resource: + properties: + schema_url: + type: keyword + ignore_above: 1024 + dropped_attributes_count: + type: long + attributes: + type: passthrough + dynamic: true + priority: 30 + time_series_dimension: true + dynamic_templates: + - complex_attributes: + path_match: + - resource.attributes.* + - scope.attributes.* + - attributes.* + match_mapping_type: object + mapping: + type: flattened diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml new file mode 100644 index 0000000000000..a32dd42e560a5 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -0,0 +1,119 @@ +--- +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for all APM data streams + managed: true +template: + mappings: + properties: + resource: + properties: + attributes: + type: passthrough + dynamic: true + priority: 30 + time_series_dimension: true + properties: + service.instance.id: + type: keyword + ignore_above: 1024 + deployment.environment: + type: keyword + ignore_above: 1024 + cloud.platform: + type: keyword + ignore_above: 1024 + container.image.tags: + type: keyword + ignore_above: 1024 + host.arch: + type: keyword + ignore_above: 1024 + process.executable.path: + type: keyword + ignore_above: 1024 + process.runtime.path: + type: keyword + ignore_above: 1024 + process.runtime.version: + type: keyword + ignore_above: 1024 + os.name: + type: keyword + ignore_above: 1024 + os.type: + type: keyword + ignore_above: 1024 + os.description: + type: keyword + ignore_above: 1024 + os.version: + type: keyword + ignore_above: 1024 + k8s.deployment.name: + type: keyword + ignore_above: 1024 + k8s.namespace.name: + type: keyword + ignore_above: 1024 + k8s.node.name: + type: keyword + ignore_above: 1024 + k8s.pod.name: + type: keyword + ignore_above: 1024 + k8s.pod.uid: + type: keyword + ignore_above: 1024 + service.node.name: + type: alias + path: resource.attributes.service.instance.id + service.environment: + type: alias + path: resource.attributes.deployment.environment + cloud.service.name: + type: alias + path: resource.attributes.cloud.platform + container.image.tag: + type: alias + path: resource.attributes.container.image.tags + host.architecture: + type: alias + path: resource.attributes.host.arch + process.executable: + type: alias + path: resource.attributes.process.executable.path + service.runtime.name: + type: alias + path: resource.attributes.process.runtime.path + service.runtime.version: + type: alias + path: resource.attributes.process.runtime.version + host.os.name: + type: alias + path: resource.attributes.os.name + host.os.platform: + type: alias + path: resource.attributes.os.type + host.os.full: + type: alias + path: resource.attributes.os.description + host.os.version: + type: alias + path: resource.attributes.os.version + kubernetes.deployment.name: + type: alias + path: resource.attributes.k8s.deployment.name + kubernetes.namespace: + type: alias + path: resource.attributes.k8s.namespace.name + kubernetes.node.name: + type: alias + path: resource.attributes.k8s.node.name + kubernetes.pod.name: + type: alias + path: resource.attributes.k8s.pod.name + kubernetes.pod.uid: + type: alias + path: resource.attributes.k8s.pod.uid + diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/tracer-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/tracer-otel@mappings.yaml new file mode 100644 index 0000000000000..c4286b3c0cccb --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/tracer-otel@mappings.yaml @@ -0,0 +1,71 @@ +--- +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for all APM data streams + managed: true +template: + mappings: + date_detection: false + dynamic: strict + properties: + data_stream.namespace: + type: constant_keyword + value: traces + trace_id: + type: keyword + trace.id: + type: alias + path: trace_id + span_id: + type: keyword + span.id: + type: alias + path: span_id + trace_state: + type: keyword + parent_span_id: + type: keyword + parent.id: + type: alias + path: parent_span_id + trace_flags: + type: byte + parent_is_remote: + type: boolean + name: + type: keyword + span.name: + type: alias + path: name + kind: + type: keyword + duration: + type: long + meta: + unit: nanos + dropped_events_count: + type: long + links: + properties: + trace_id: + type: keyword + span_id: + type: keyword + trace_state: + type: keyword + attributes: + type: object + subobjects: false + dynamic: true + dropped_attributes_count: + type: long + trace_flags: + type: byte + dropped_links_count: + type: long + status: + properties: + message: + type: keyword + status_code: + type: keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml new file mode 100644 index 0000000000000..f72ccb2e68a27 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml @@ -0,0 +1,13 @@ +--- +version: ${xpack.oteldata.template.version} +_meta: + description: default policy for the traces index template installed by x-pack + managed: true +template: + lifecycle: + phases: + hot: + actions: + rollover: + max_primary_shard_size: 50gb + max_age: 30d diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml new file mode 100644 index 0000000000000..1e06cfa58a25e --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml @@ -0,0 +1,17 @@ +--- +version: ${xpack.oteldata.template.version} +_meta: + description: Default mappings for the traces index template installed by x-pack + managed: true +template: + mappings: + properties: + "@timestamp": + type: date + data_stream.type: + type: constant_keyword + value: traces + data_stream.dataset: + type: constant_keyword + data_stream.namespace: + type: constant_keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml new file mode 100644 index 0000000000000..c13f9ea42347d --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml @@ -0,0 +1,12 @@ +version: ${xpack.oteldata.template.version} +_meta: + description: Default settings for OpenTelemetry traces + managed: true +template: + settings: + index: + lifecycle: + name: traces + codec: best_compression + mapping: + ignore_malformed: true diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/README.md b/x-pack/plugin/otel-data/src/main/resources/index-templates/README.md new file mode 100644 index 0000000000000..99f4f4218ca90 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/README.md @@ -0,0 +1,7 @@ +Mappings and settings that must not be overridden are to be defined +in the index templates, as these will take precedence over component +template mappings and settings. This includes: + + - `index.default_pipeline` + - `index.final_pipeline` + - values for `data_stream.*` diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml new file mode 100644 index 0000000000000..dbb29e73ce656 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml @@ -0,0 +1,21 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["logs-*.otel-*"] +priority: 120 +data_stream: {} +allow_auto_create: true +_meta: + description: default logs template installed by x-pack + managed: true +composed_of: + - logs@mappings + - logs@settings + - otel@mappings + - logs-otel@mappings + - semconv-resource-to-ecs@mappings + - logs@custom + - logs-otel@custom + - ecs@mappings +ignore_missing_component_templates: + - logs@custom + - logs-otel@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml new file mode 100644 index 0000000000000..df24e3365700e --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -0,0 +1,23 @@ +--- +index_patterns: ["metrics-*.otel-*"] +priority: 120 +data_stream: {} +allow_auto_create: true +_meta: + description: default logs template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom +template: + settings: + index: + mode: time_series diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml new file mode 100644 index 0000000000000..f5f143be8ee00 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -0,0 +1,21 @@ +--- +_meta: + description: default logs template installed by x-pack + managed: true +index_patterns: + - "traces-*.otel-*" +priority: 120 +data_stream: {} +composed_of: + - traces@mappings + - traces@settings + - otel@mappings + - traces-otel@mappings + - semconv-resource-to-ecs@mappings + - traces@custom + - traces-otel@custom + - ecs@mappings +ignore_missing_component_templates: + - traces@custom + - traces-otel@custom +allow_auto_create: true diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml new file mode 100644 index 0000000000000..0812be5daf4ae --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -0,0 +1,20 @@ +# "version" holds the version of the templates and ingest pipelines installed +# by xpack-plugin apm-data. This must be increased whenever an existing template or +# pipeline is changed, in order for it to be updated on Elasticsearch upgrade. +version: 1 + +component-templates: + - otel@mappings + - logs-otel@mappings + - semconv-resource-to-ecs@mappings + - ecs-tsdb@mappings + - metrics-otel@mappings + - tracer-otel@mappings + # Data lifecycle. + - traces@lifecycle + + - traces@settings +index-templates: + - logs-otel@template + - metrics-otel@template + - traces-otel@template From d5861bb054b16a1c6e97876a368f3c016f8fb12c Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 18 Jul 2024 17:34:49 +0200 Subject: [PATCH 02/51] Fix traces-otel template --- ...el@mappings.yaml => traces-otel@mappings.yaml} | 0 .../component-templates/traces@lifecycle.yaml | 13 ------------- .../component-templates/traces@settings.yaml | 6 +++--- .../index-templates/traces-otel@template.yaml | 15 ++++++++------- .../otel-data/src/main/resources/resources.yaml | 8 ++------ .../xpack/security/authz/AuthorizationUtils.java | 2 ++ 6 files changed, 15 insertions(+), 29 deletions(-) rename x-pack/plugin/otel-data/src/main/resources/component-templates/{tracer-otel@mappings.yaml => traces-otel@mappings.yaml} (100%) delete mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/tracer-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml similarity index 100% rename from x-pack/plugin/otel-data/src/main/resources/component-templates/tracer-otel@mappings.yaml rename to x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml deleted file mode 100644 index f72ccb2e68a27..0000000000000 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@lifecycle.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -version: ${xpack.oteldata.template.version} -_meta: - description: default policy for the traces index template installed by x-pack - managed: true -template: - lifecycle: - phases: - hot: - actions: - rollover: - max_primary_shard_size: 50gb - max_age: 30d diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml index c13f9ea42347d..362f3110b7838 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml @@ -4,9 +4,9 @@ _meta: managed: true template: settings: - index: - lifecycle: - name: traces +# index: +# lifecycle: +# name: traces codec: best_compression mapping: ignore_malformed: true diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index f5f143be8ee00..55921202835b7 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -1,14 +1,15 @@ --- +index_patterns: ["traces-*.otel-*"] +priority: 120 +data_stream: {} +allow_auto_create: true _meta: description: default logs template installed by x-pack managed: true -index_patterns: - - "traces-*.otel-*" -priority: 120 -data_stream: {} composed_of: - - traces@mappings - - traces@settings +# It's TBD how we proceed with these - potentially this could be shared with apm +# - traces@mappings +# - traces@settings - otel@mappings - traces-otel@mappings - semconv-resource-to-ecs@mappings @@ -18,4 +19,4 @@ composed_of: ignore_missing_component_templates: - traces@custom - traces-otel@custom -allow_auto_create: true + diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index 0812be5daf4ae..8760b5922d3db 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,5 +1,5 @@ # "version" holds the version of the templates and ingest pipelines installed -# by xpack-plugin apm-data. This must be increased whenever an existing template or +# by xpack-plugin otel-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. version: 1 @@ -9,11 +9,7 @@ component-templates: - semconv-resource-to-ecs@mappings - ecs-tsdb@mappings - metrics-otel@mappings - - tracer-otel@mappings - # Data lifecycle. - - traces@lifecycle - - - traces@settings + - traces-otel@mappings index-templates: - logs-otel@template - metrics-otel@template diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index 194440722545a..4173f3db45409 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -39,6 +39,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.LOGSTASH_MANAGEMENT_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.OTEL_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.PROFILING_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ROLLUP_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.SEARCHABLE_SNAPSHOTS_ORIGIN; @@ -151,6 +152,7 @@ public static void switchUserBasedOnActionOriginAndExecute( case INGEST_ORIGIN: case PROFILING_ORIGIN: case APM_ORIGIN: + case OTEL_ORIGIN: case STACK_ORIGIN: case SEARCHABLE_SNAPSHOTS_ORIGIN: case LOGSTASH_MANAGEMENT_ORIGIN: From e58d3590c23b3188ed37f04a5330647660f26c27 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 18 Jul 2024 23:46:40 +0200 Subject: [PATCH 03/51] Adding first yml tests --- .../xpack/core/XPackSettings.java | 3 ++ .../xpack/oteldata/OTelPlugin.java | 4 +- .../component-templates/otel@mappings.yaml | 8 ++- .../metrics-otel@template.yaml | 1 + .../index-templates/traces-otel@template.yaml | 1 + .../xpack/oteldata/OTelYamlTestSuiteIT.java | 52 +++++++++++++++++++ .../resources/rest-api-spec/test/10_otel.yml | 38 ++++++++++++++ 7 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index d33b2aecdab04..90905df12a2cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -93,6 +93,9 @@ public Iterator> settings() { /** Setting for enabling or disabling APM Data. Defaults to true. */ public static final Setting APM_DATA_ENABLED = Setting.boolSetting("xpack.apm_data.enabled", true, Setting.Property.NodeScope); + /** Setting for enabling or disabling OTel Data. Defaults to true. */ + public static final Setting OTEL_DATA_ENABLED = Setting.boolSetting("xpack.otel_data.enabled", true, Setting.Property.NodeScope); + /** Setting for enabling or disabling enterprise search. Defaults to true. */ public static final Setting ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting( "xpack.ent_search.enabled", diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 905eceecc7fa8..241025d528967 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -40,12 +40,12 @@ public class OTelPlugin extends Plugin implements ActionPlugin { ); public OTelPlugin(Settings settings) { - this.enabled = XPackSettings.APM_DATA_ENABLED.get(settings); + this.enabled = XPackSettings.OTEL_DATA_ENABLED.get(settings); } @Override public Collection createComponents(PluginServices services) { - logger.info("APM ingest plugin is {}", enabled ? "enabled" : "disabled"); + logger.info("OTel ingest plugin is {}", enabled ? "enabled" : "disabled"); Settings settings = services.environment().settings(); ClusterService clusterService = services.clusterService(); registry.set( diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 0a6e571f415f5..fb4ee0c28139b 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -28,8 +28,12 @@ template: name: type: keyword ignore_above: 1024 - version: - type: version +# TODO: +#Causes error during YAML test +# [2024-07-19T04:36:51,833][ERROR][o.e.x.c.t.IndexTemplateRegistry] [test-cluster-0] error adding index template [otel@mappings] for [otel] org.elasticsearch.index.mapper.MapperParsingException: Failed to parse mapping: No handler for type [version] declared on field [version] +# at org.elasticsearch.server@8.16.0-SNAPSHOT/org.elasticsearch.index.mapper.MapperService.parseMapping(MapperService.java:610) +# version: +# type: version schema_url: type: keyword ignore_above: 1024 diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index df24e3365700e..a8a257623c876 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -1,4 +1,5 @@ --- +version: ${xpack.oteldata.template.version} index_patterns: ["metrics-*.otel-*"] priority: 120 data_stream: {} diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index 55921202835b7..c508ff9c38c2d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -1,4 +1,5 @@ --- +version: ${xpack.oteldata.template.version} index_patterns: ["traces-*.otel-*"] priority: 120 data_stream: {} diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java new file mode 100644 index 0000000000000..c3caf9be3b263 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.oteldata; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class OTelYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("constant-keyword") + .module("counted-keyword") + .module("data-streams") + .module("ingest-common") + .module("ingest-geoip") + .module("ingest-user-agent") + .module("lang-mustache") + .module("mapper-extras") + .module("wildcard") + .module("x-pack-analytics") + .module("x-pack-otel-data") + .module("x-pack-aggregate-metric") + .module("x-pack-ilm") + .module("x-pack-stack") + .setting("ingest.geoip.downloader.enabled", "false") + .build(); + + public OTelYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml new file mode 100644 index 0000000000000..72b7a127dcd02 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml @@ -0,0 +1,38 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + +--- +"Test traces-otel* template installation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.get_index_template: + name: traces-otel* + - length: {index_templates: 1} + - contains: {index_templates: {name: traces-otel@template}} + +--- +"Test metrics-otel* template installation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.get_index_template: + name: metrics-otel* + - length: {index_templates: 1} + - contains: {index_templates: {name: metrics-otel@template}} + +--- +"Test logs-otel* template installation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.get_index_template: + name: logs-otel* + - length: {index_templates: 1} + - contains: {index_templates: {name: logs-otel@template}} From 4b7414cbd1ebb45ef6ff7ae1ea7c0fad7730de3b Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 19 Jul 2024 12:24:36 +0200 Subject: [PATCH 04/51] Base APMIndexTemplateRegistry on YamlTemplateRegistry --- .../apmdata/APMIndexTemplateRegistry.java | 173 ++---------------- .../xpack/apmdata/ResourceUtils.java | 43 ----- .../apmdata/YamlIngestPipelineConfig.java | 36 ---- .../template/YamlIngestPipelineConfig.java | 16 +- .../core/template/YamlTemplateRegistry.java | 18 +- 5 files changed, 31 insertions(+), 255 deletions(-) delete mode 100644 x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java delete mode 100644 x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 04b0257f4180a..7851ed4ac3ab0 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -7,188 +7,37 @@ package org.elasticsearch.xpack.apmdata; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.yaml.YamlXContent; -import org.elasticsearch.xpack.core.ClientHelper; -import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; -import org.elasticsearch.xpack.core.template.IngestPipelineConfig; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.elasticsearch.xpack.apmdata.ResourceUtils.APM_TEMPLATE_VERSION_VARIABLE; -import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadResource; -import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8; +import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; /** * Creates all index templates and ingest pipelines that are required for using Elastic APM. */ -public class APMIndexTemplateRegistry extends IndexTemplateRegistry { - private static final Logger logger = LogManager.getLogger(APMIndexTemplateRegistry.class); - // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a - // dependency to the data-streams module just for this - public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); - private final int version; - - private final Map componentTemplates; - private final Map composableIndexTemplates; - private final List ingestPipelines; - private final FeatureService featureService; - private volatile boolean enabled; - - @SuppressWarnings("unchecked") - public APMIndexTemplateRegistry( - Settings nodeSettings, - ClusterService clusterService, - ThreadPool threadPool, - Client client, - NamedXContentRegistry xContentRegistry, - FeatureService featureService - ) { - super(nodeSettings, clusterService, threadPool, client, xContentRegistry); - - try { - final Map apmResources = XContentHelper.convertToMap( - YamlXContent.yamlXContent, - loadResource("/resources.yaml"), - false - ); - version = (((Number) apmResources.get("version")).intValue()); - final List componentTemplateNames = (List) apmResources.get("component-templates"); - final List indexTemplateNames = (List) apmResources.get("index-templates"); - final List ingestPipelineConfigs = (List) apmResources.get("ingest-pipelines"); - - componentTemplates = componentTemplateNames.stream() - .map(o -> (String) o) - .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); - composableIndexTemplates = indexTemplateNames.stream() - .map(o -> (String) o) - .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); - ingestPipelines = ingestPipelineConfigs.stream().map(o -> (Map>) o).map(map -> { - Map.Entry> pipelineConfig = map.entrySet().iterator().next(); - return loadIngestPipeline(pipelineConfig.getKey(), version, (List) pipelineConfig.getValue().get("dependencies")); - }).collect(Collectors.toList()); - this.featureService = featureService; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public int getVersion() { - return version; - } - - void setEnabled(boolean enabled) { - logger.info("APM index template registry is {}", enabled ? "enabled" : "disabled"); - this.enabled = enabled; - } - - public boolean isEnabled() { - return enabled; - } - - public void close() { - clusterService.removeListener(this); - } - - @Override - protected String getOrigin() { - return ClientHelper.APM_ORIGIN; - } - - @Override - protected boolean isClusterReady(ClusterChangedEvent event) { - // Ensure current version of the components are installed only after versions that support data stream lifecycle - // due to the use of the feature in all the `@lifecycle` component templates - return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE); - } +public class APMIndexTemplateRegistry extends YamlTemplateRegistry { - @Override - protected boolean requiresMasterNode() { - return true; - } + public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version"; - @Override - protected Map getComponentTemplateConfigs() { - if (enabled) { - return componentTemplates; - } else { - return Map.of(); - } + public APMIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, NamedXContentRegistry xContentRegistry, FeatureService featureService) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); } @Override - protected Map getComposableTemplateConfigs() { - if (enabled) { - return composableIndexTemplates; - } else { - return Map.of(); - } + public String getName() { + return "apm"; } @Override - protected List getIngestPipelines() { - if (enabled) { - return ingestPipelines; - } else { - return Collections.emptyList(); - } - } - - private static ComponentTemplate loadComponentTemplate(String name, int version) { - try { - final byte[] content = loadVersionedResourceUTF8("/component-templates/" + name + ".yaml", version); - try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { - return ComponentTemplate.parse(parser); - } - } catch (Exception e) { - throw new RuntimeException("failed to load APM Ingest plugin's component template: " + name, e); - } - } - - private static ComposableIndexTemplate loadIndexTemplate(String name, int version) { - try { - final byte[] content = loadVersionedResourceUTF8("/index-templates/" + name + ".yaml", version); - try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { - return ComposableIndexTemplate.parse(parser); - } - } catch (Exception e) { - throw new RuntimeException("failed to load APM Ingest plugin's index template: " + name, e); - } - } - - private static IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { - if (dependencies == null) { - dependencies = Collections.emptyList(); - } - return new YamlIngestPipelineConfig( - name, - "/ingest-pipelines/" + name + ".yaml", - version, - APM_TEMPLATE_VERSION_VARIABLE, - dependencies - ); + protected String getVersionProperty() { + return APM_TEMPLATE_VERSION_VARIABLE; } @Override - protected boolean applyRolloverAfterTemplateV2Upgrade() { - return true; + protected String getOrigin() { + return "apm"; } } diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java deleted file mode 100644 index 1e6a9a9998a82..0000000000000 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.apmdata; - -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.xpack.core.template.TemplateUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -public class ResourceUtils { - - public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version"; - - static byte[] loadVersionedResourceUTF8(String name, int version) { - return loadVersionedResourceUTF8(name, version, Map.of()); - } - - static byte[] loadVersionedResourceUTF8(String name, int version, Map variables) { - try { - String content = loadResource(name); - content = TemplateUtils.replaceVariables(content, String.valueOf(version), APM_TEMPLATE_VERSION_VARIABLE, variables); - return content.getBytes(StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - static String loadResource(String name) throws IOException { - InputStream is = APMIndexTemplateRegistry.class.getResourceAsStream(name); - if (is == null) { - throw new IOException("Resource [" + name + "] not found in classpath."); - } - return Streams.readFully(is).utf8ToString(); - } -} diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java deleted file mode 100644 index de1b715dd138d..0000000000000 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.apmdata; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.template.IngestPipelineConfig; - -import java.util.List; - -import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8; - -/** - * An APM-plugin-specific implementation that loads ingest pipelines in yaml format from a local resources repository - */ -public class YamlIngestPipelineConfig extends IngestPipelineConfig { - public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies) { - super(id, resource, version, versionProperty, dependencies); - } - - @Override - public XContentType getXContentType() { - return XContentType.YAML; - } - - @Override - public BytesReference loadConfig() { - return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version, variables)); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java index d88b17bed541a..e262927b6e17b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java @@ -8,13 +8,18 @@ package org.elasticsearch.xpack.core.template; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.xcontent.XContentType; import java.util.List; +import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; + public class YamlIngestPipelineConfig extends IngestPipelineConfig { - public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies) { + private final Class clazz; + public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies, Class clazz) { super(id, resource, version, versionProperty, dependencies); + this.clazz = clazz; } @Override @@ -24,7 +29,12 @@ public XContentType getXContentType() { @Override public BytesReference loadConfig() { - // return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version, variables)); - return null; // TODO + return new BytesArray(loadVersionedResourceUTF8(clazz, "/ingest-pipelines/" + id + ".yaml", version, versionProperty, variables)); + //return null; // TODO + + //orig: + // return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version, variables)); } + + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index 7f528bf9d05b1..3a8da6f40f250 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -62,17 +62,12 @@ public YamlTemplateRegistry( super(nodeSettings, clusterService, threadPool, client, xContentRegistry); try { - - var clazz = this.getClass(); - logger.info("========= classname: " + clazz.getName()); - final Map resources = XContentHelper.convertToMap( YamlXContent.yamlXContent, loadResource(this.getClass(), "/resources.yaml"), false ); version = (((Number) resources.get("version")).intValue()); - logger.info("========= version: " + version); final List componentTemplateNames = (List) resources.get("component-templates"); final List indexTemplateNames = (List) resources.get("index-templates"); @@ -136,7 +131,7 @@ protected boolean requiresMasterNode() { } @Override - protected Map getComponentTemplateConfigs() { + public Map getComponentTemplateConfigs() { if (enabled) { return componentTemplates; } else { @@ -145,7 +140,7 @@ protected Map getComponentTemplateConfigs() { } @Override - protected Map getComposableTemplateConfigs() { + public Map getComposableTemplateConfigs() { if (enabled) { return composableIndexTemplates; } else { @@ -154,7 +149,7 @@ protected Map getComposableTemplateConfigs() { } @Override - protected List getIngestPipelines() { + public List getIngestPipelines() { if (enabled) { return ingestPipelines; } else { @@ -188,7 +183,7 @@ private ComposableIndexTemplate loadIndexTemplate(String name, int version) { } } - private static IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { + private IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { if (dependencies == null) { dependencies = Collections.emptyList(); } @@ -196,8 +191,9 @@ private static IngestPipelineConfig loadIngestPipeline(String name, int version, name, "/ingest-pipelines/" + name + ".yaml", version, - "TODO", //APM_TEMPLATE_VERSION_VARIABLE //TODO - dependencies + getVersionProperty(), + dependencies, + this.getClass() ); } From b65bb902b72d144d947134585fb59b7f4c4f4f46 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 19 Jul 2024 12:26:43 +0200 Subject: [PATCH 05/51] Update OTelPlugin.java --- .../main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 241025d528967..64faaa572a7e0 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -34,7 +34,7 @@ public class OTelPlugin extends Plugin implements ActionPlugin { // This setting will be ignored if the plugin is disabled. static final Setting OTEL_DATA_REGISTRY_ENABLED = Setting.boolSetting( "xpack.otel_data.registry.enabled", - true, + false, Setting.Property.NodeScope, Setting.Property.Dynamic ); From fc82339bef3af90c5218793b7547aba2d42cabe4 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 19 Jul 2024 12:28:07 +0200 Subject: [PATCH 06/51] Update APMIndexTemplateRegistry.java --- .../elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 7851ed4ac3ab0..4fb17e218114a 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -13,6 +13,7 @@ import org.elasticsearch.features.FeatureService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; /** @@ -38,6 +39,6 @@ protected String getVersionProperty() { @Override protected String getOrigin() { - return "apm"; + return ClientHelper.APM_ORIGIN; } } From 72a6769503342afa5c2d0b8403096e5590f0cc28 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 19 Jul 2024 12:41:54 +0200 Subject: [PATCH 07/51] Update YamlIngestPipelineConfig.java --- .../xpack/core/template/YamlIngestPipelineConfig.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java index e262927b6e17b..84b3703036194 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java @@ -30,10 +30,6 @@ public XContentType getXContentType() { @Override public BytesReference loadConfig() { return new BytesArray(loadVersionedResourceUTF8(clazz, "/ingest-pipelines/" + id + ".yaml", version, versionProperty, variables)); - //return null; // TODO - - //orig: - // return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version, variables)); } From 9cd50eedbf785126cb1137c68ade8cfbfc3ecaa1 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 22 Jul 2024 20:03:23 +0200 Subject: [PATCH 08/51] Adding traces tests --- .../APMIndexTemplateRegistryTests.java | 3 +- .../xpack/oteldata/OTelYamlTestSuiteIT.java | 1 + .../rest-api-spec/test/20_metic_tests.yml | 24 +++++++ .../rest-api-spec/test/20_traces_tests.yml | 69 +++++++++++++++++++ 4 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index e9f0775836c71..1d6faa0f403d4 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -75,7 +75,6 @@ public class APMIndexTemplateRegistryTests extends ESTestCase { private APMIndexTemplateRegistry apmIndexTemplateRegistry; private StackTemplateRegistryAccessor stackTemplateRegistryAccessor; - private ClusterService clusterService; private ThreadPool threadPool; private VerifyingClient client; @@ -89,7 +88,7 @@ public void createRegistryAndClient() { threadPool = new TestThreadPool(this.getClass().getName()); client = new VerifyingClient(threadPool); - clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings); + ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings); FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures())); stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor( new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY, featureService) diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java index c3caf9be3b263..373cb46b3a482 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java +++ b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java @@ -34,6 +34,7 @@ public class OTelYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .module("x-pack-ilm") .module("x-pack-stack") .setting("ingest.geoip.downloader.enabled", "false") + .setting("xpack.otel_data.registry.enabled", "true") .build(); public OTelYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml new file mode 100644 index 0000000000000..7f06d530e4006 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -0,0 +1,24 @@ + +--- +setup: + - do: + cluster.health: + wait_for_events: languid + +--- +"Test push service overview metric": + - skip: + awaits_fix: "TODO: metrics-otel@template uses TSDB, so @timestamp needs to be up-to-date. Q: How to do that in this syntax?" + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - is_false: errors + + - do: + search: + index: metrics-generic.otel-default + - length: { hits.hits: 1 } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml new file mode 100644 index 0000000000000..0bbbea0e41e2c --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -0,0 +1,69 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid +--- +"Test pushing simple trace": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - is_false: errors + - do: + search: + index: traces-generic.otel-default + - length: { hits.hits: 1 } + +--- +"Reject invalid top level field": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"invalid_top_level_field": "foo", "@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - is_true: errors + - do: + search: + index: traces-generic.otel-default + - length: { hits.hits: 0 } +--- +"Query resource attributes as top level": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - is_false: errors + - do: + search: + index: traces-generic.otel-default + body: + fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} + +--- +"Query attributes as top level": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - is_false: errors + - do: + search: + index: traces-generic.otel-default + body: + fields: ["db.type", "db.name", "db.operation", "db.statement"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"db.type": ["mssql"], "db.name": ["foo"], "db.operation": ["SELECT"], "db.statement": ["SELECT * FROM wuser_table"] }} From 953968ff7d38364484d644146f817e8ea4322ee7 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Mon, 22 Jul 2024 20:04:34 +0200 Subject: [PATCH 09/51] Update x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml Co-authored-by: Felix Barnsteiner --- .../resources/component-templates/ecs-tsdb@mappings.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml index eed2b0c605175..41f380a2a402d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -5,14 +5,6 @@ _meta: template: mappings: dynamic_templates: - - ecs_ip: - mapping: - type: ip - path_match: - - "ip" - - "*.ip" - - "*_ip" - match_mapping_type: string - all_strings_to_keywords: mapping: ignore_above: 1024 From 6206ab3f7fa6f5cc7a460f8a94719c853d2fe895 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 25 Jul 2024 17:27:20 +0200 Subject: [PATCH 10/51] Add mapper-version --- x-pack/plugin/otel-data/build.gradle | 1 + .../resources/component-templates/otel@mappings.yaml | 10 ++++------ .../xpack/oteldata/OTelYamlTestSuiteIT.java | 1 + 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index fe030adee26dc..f56efe21acccc 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -34,4 +34,5 @@ dependencies { clusterModules project(xpackModule('mapper-counted-keyword')) clusterModules project(xpackModule('stack')) clusterModules project(xpackModule('wildcard')) + clusterModules project(xpackModule('mapper-version')) } diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index fb4ee0c28139b..bf5d4851d0a83 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -23,17 +23,15 @@ template: time_series_dimension: true dropped_attributes_count: type: long + processor.event: + type: keyword scope: properties: name: type: keyword ignore_above: 1024 -# TODO: -#Causes error during YAML test -# [2024-07-19T04:36:51,833][ERROR][o.e.x.c.t.IndexTemplateRegistry] [test-cluster-0] error adding index template [otel@mappings] for [otel] org.elasticsearch.index.mapper.MapperParsingException: Failed to parse mapping: No handler for type [version] declared on field [version] -# at org.elasticsearch.server@8.16.0-SNAPSHOT/org.elasticsearch.index.mapper.MapperService.parseMapping(MapperService.java:610) -# version: -# type: version + version: + type: version schema_url: type: keyword ignore_above: 1024 diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java index 373cb46b3a482..4a5f7d03b12a2 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java +++ b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java @@ -33,6 +33,7 @@ public class OTelYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .module("x-pack-aggregate-metric") .module("x-pack-ilm") .module("x-pack-stack") + .module("mapper-version") .setting("ingest.geoip.downloader.enabled", "false") .setting("xpack.otel_data.registry.enabled", "true") .build(); From c7ee94c7fafa7489a03873597d4d94e1287997ce Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 25 Jul 2024 17:32:52 +0200 Subject: [PATCH 11/51] Fix code-style --- .../xpack/apmdata/APMIndexTemplateRegistry.java | 3 ++- .../java/org/elasticsearch/xpack/core/XPackSettings.java | 3 ++- .../org/elasticsearch/xpack/core/template/ResourceUtils.java | 3 ++- .../org/elasticsearch/xpack/core/template/TemplateUtils.java | 5 ----- .../xpack/core/template/YamlIngestPipelineConfig.java | 5 ++--- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 4fb17e218114a..3fe101ba414c9 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -23,7 +23,8 @@ public class APMIndexTemplateRegistry extends YamlTemplateRegistry { public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version"; - public APMIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, NamedXContentRegistry xContentRegistry, FeatureService featureService) { + public APMIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, + NamedXContentRegistry xContentRegistry, FeatureService featureService) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 90905df12a2cb..0637d27556902 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -94,7 +94,8 @@ public Iterator> settings() { public static final Setting APM_DATA_ENABLED = Setting.boolSetting("xpack.apm_data.enabled", true, Setting.Property.NodeScope); /** Setting for enabling or disabling OTel Data. Defaults to true. */ - public static final Setting OTEL_DATA_ENABLED = Setting.boolSetting("xpack.otel_data.enabled", true, Setting.Property.NodeScope); + public static final Setting OTEL_DATA_ENABLED = + Setting.boolSetting("xpack.otel_data.enabled", true, Setting.Property.NodeScope); /** Setting for enabling or disabling enterprise search. Defaults to true. */ public static final Setting ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java index 2ca3c5ec4fa02..bb080e79d35da 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java @@ -18,7 +18,8 @@ static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version return loadVersionedResourceUTF8(clazz, name, version, versionProperty, Map.of()); } - static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty, Map variables) { + static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty, + Map variables) { try { String content = loadResource(clazz, name); content = TemplateUtils.replaceVariables(content, String.valueOf(version), versionProperty, variables); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java index f9794000ee295..ba9639d3d5156 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java @@ -10,15 +10,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.template.resources.TemplateResources; -import org.elasticsearch.xpack.core.watcher.input.Input; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java index 84b3703036194..d9fdfff90cc68 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java @@ -17,7 +17,8 @@ public class YamlIngestPipelineConfig extends IngestPipelineConfig { private final Class clazz; - public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies, Class clazz) { + public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies, + Class clazz) { super(id, resource, version, versionProperty, dependencies); this.clazz = clazz; } @@ -31,6 +32,4 @@ public XContentType getXContentType() { public BytesReference loadConfig() { return new BytesArray(loadVersionedResourceUTF8(clazz, "/ingest-pipelines/" + id + ".yaml", version, versionProperty, variables)); } - - } From effc9c3b7033a20e51c228d5dc3f7e97df31c2ef Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 25 Jul 2024 17:34:56 +0200 Subject: [PATCH 12/51] Rename `status.status_code` to `status.code` --- .../component-templates/traces-otel@mappings.yaml | 2 +- .../resources/rest-api-spec/test/20_traces_tests.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index c4286b3c0cccb..0a4e1c1f2bc9f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -67,5 +67,5 @@ template: properties: message: type: keyword - status_code: + code: type: keyword diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index 0bbbea0e41e2c..ca1f461428f9c 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -11,7 +11,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: @@ -26,7 +26,7 @@ setup: refresh: true body: - create: {} - - '{"invalid_top_level_field": "foo", "@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - '{"invalid_top_level_field": "foo", "@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_true: errors - do: search: @@ -40,7 +40,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: @@ -58,7 +58,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"status_code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: From ae4fc40e59021fdf4f127b8c172c4308cdc72791 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 25 Jul 2024 18:14:48 +0200 Subject: [PATCH 13/51] Update otel@mappings.yaml Revert back to date due to missing support in ES|QL for date_nanos --- .../src/main/resources/component-templates/otel@mappings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index bf5d4851d0a83..a529dba006bb4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -9,7 +9,7 @@ template: dynamic: strict properties: "@timestamp": - type: date_nanos + type: date data_stream.type: type: constant_keyword data_stream.dataset: From 909a8dd6d9b109e486a123ffc9445399cf48e6da Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 25 Jul 2024 18:15:31 +0200 Subject: [PATCH 14/51] Move dynamic_templates to metrics@mappings in core --- .../src/main/resources/metrics@mappings.json | 62 +++++++++++++++++++ .../metrics-otel@mappings.yaml | 35 ----------- .../metrics-otel@template.yaml | 1 + 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json index b4aa999697632..d4ddbf8f31de2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json @@ -43,6 +43,68 @@ "default_metric": "value_count" } } + }, + { + "histogram": { + "mapping": { + "type": "histogram", + "ignore_malformed": true + } + } + }, + { + "counter_long": { + "mapping": { + "type": "long", + "time_series_metric": "counter", + "ignore_malformed": true + } + } + }, + { + "gauge_long": { + "mapping": { + "type": "long", + "time_series_metric": "gauge", + "ignore_malformed": true + } + } + }, + { + "counter_double": { + "mapping": { + "type": "double", + "time_series_metric": "counter", + "ignore_malformed": true + } + } + }, + { + "gauge_double": { + "mapping": { + "type": "double", + "time_series_metric": "gauge", + "ignore_malformed": true + } + } + }, + { + "summary_gauge": { + "mapping": { + "type": "aggregate_metric_double", + "time_series_metric": "gauge", + "ignore_malformed": true + } + } + }, + { + "summary_counter": { + "mapping": { + "type": "aggregate_metric_double", + "time_series_metric": "counter", + "ignore_malformed": true + } + } } ], "properties": { diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index b8751ef60aca6..6fc0c7274b97d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -18,38 +18,3 @@ template: type: keyword time_series_dimension: true ignore_above: 1024 - dynamic_templates: - - histogram: - mapping: - type: histogram - ignore_malformed: true - - counter_long: - mapping: - type: long - time_series_metric: counter - ignore_malformed: true - - gauge_long: - mapping: - type: long - time_series_metric: gauge - ignore_malformed: true - - counter_double: - mapping: - type: double - time_series_metric: counter - ignore_malformed: true - - gauge_double: - mapping: - type: double - time_series_metric: gauge - ignore_malformed: true - - summary_gauge: - mapping: - type: aggregate_metric_double - time_series_metric: gauge - ignore_malformed: true - - summary_counter: - mapping: - type: aggregate_metric_double - time_series_metric: counter - ignore_malformed: true diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index a8a257623c876..2926b36fd0432 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -10,6 +10,7 @@ _meta: composed_of: - metrics@tsdb-settings - otel@mappings + - metrics@mappings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom From 68ff941844613c3a4d138f4f0798ca509287074e Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 25 Jul 2024 18:56:54 +0200 Subject: [PATCH 15/51] Run gradlew :x-pack:plugin:core:spotlessApply --- .../apmdata/APMIndexTemplateRegistry.java | 10 ++++- .../xpack/core/XPackSettings.java | 7 +++- .../xpack/core/template/ResourceUtils.java | 14 ++++--- .../template/YamlIngestPipelineConfig.java | 13 +++++-- .../core/template/YamlTemplateRegistry.java | 39 +++++++++++++------ .../xpack/oteldata/OTelPlugin.java | 1 - .../oteldata/OtelIndexTemplateRegistry.java | 10 ++++- 7 files changed, 67 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 3fe101ba414c9..10c02fd7f1036 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -23,8 +23,14 @@ public class APMIndexTemplateRegistry extends YamlTemplateRegistry { public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version"; - public APMIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, - NamedXContentRegistry xContentRegistry, FeatureService featureService) { + public APMIndexTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + FeatureService featureService + ) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 0637d27556902..f76b0d2bb6d8d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -94,8 +94,11 @@ public Iterator> settings() { public static final Setting APM_DATA_ENABLED = Setting.boolSetting("xpack.apm_data.enabled", true, Setting.Property.NodeScope); /** Setting for enabling or disabling OTel Data. Defaults to true. */ - public static final Setting OTEL_DATA_ENABLED = - Setting.boolSetting("xpack.otel_data.enabled", true, Setting.Property.NodeScope); + public static final Setting OTEL_DATA_ENABLED = Setting.boolSetting( + "xpack.otel_data.enabled", + true, + Setting.Property.NodeScope + ); /** Setting for enabling or disabling enterprise search. Defaults to true. */ public static final Setting ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java index bb080e79d35da..9840535989a7c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java @@ -12,14 +12,18 @@ import java.nio.charset.StandardCharsets; import java.util.Map; -public class ResourceUtils -{ - static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty) { +public class ResourceUtils { + static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty) { return loadVersionedResourceUTF8(clazz, name, version, versionProperty, Map.of()); } - static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty, - Map variables) { + static byte[] loadVersionedResourceUTF8( + Class clazz, + String name, + int version, + String versionProperty, + Map variables + ) { try { String content = loadResource(clazz, name); content = TemplateUtils.replaceVariables(content, String.valueOf(version), versionProperty, variables); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java index d9fdfff90cc68..0cb69b490a73a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java @@ -7,18 +7,25 @@ package org.elasticsearch.xpack.core.template; - import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.xcontent.XContentType; + import java.util.List; import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; public class YamlIngestPipelineConfig extends IngestPipelineConfig { private final Class clazz; - public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies, - Class clazz) { + + public YamlIngestPipelineConfig( + String id, + String resource, + int version, + String versionProperty, + List dependencies, + Class clazz + ) { super(id, resource, version, versionProperty, dependencies); this.clazz = clazz; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index 3a8da6f40f250..7471f722261bf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -78,17 +78,24 @@ public YamlTemplateRegistry( .stream() .map(o -> (String) o) .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); - composableIndexTemplates = Optional.ofNullable(indexTemplateNames) + composableIndexTemplates = Optional.ofNullable(indexTemplateNames) .orElse(Collections.emptyList()) .stream() .map(o -> (String) o) .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); - ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) + ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) .orElse(Collections.emptyList()) - .stream().map(o -> (Map>) o).map(map -> { - Map.Entry> pipelineConfig = map.entrySet().iterator().next(); - return loadIngestPipeline(pipelineConfig.getKey(), version, (List) pipelineConfig.getValue().get("dependencies")); - }).collect(Collectors.toList()); + .stream() + .map(o -> (Map>) o) + .map(map -> { + Map.Entry> pipelineConfig = map.entrySet().iterator().next(); + return loadIngestPipeline( + pipelineConfig.getKey(), + version, + (List) pipelineConfig.getValue().get("dependencies") + ); + }) + .collect(Collectors.toList()); this.featureService = featureService; } catch (IOException e) { throw new RuntimeException(e); @@ -161,25 +168,33 @@ public List getIngestPipelines() { private ComponentTemplate loadComponentTemplate(String name, int version) { try { - final byte[] content = loadVersionedResourceUTF8(this.getClass(), "/component-templates/" - + name + ".yaml", version, getVersionProperty()); + final byte[] content = loadVersionedResourceUTF8( + this.getClass(), + "/component-templates/" + name + ".yaml", + version, + getVersionProperty() + ); try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { return ComponentTemplate.parse(parser); } } catch (Exception e) { - throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); + throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); } } private ComposableIndexTemplate loadIndexTemplate(String name, int version) { try { - final byte[] content = loadVersionedResourceUTF8(this.getClass(), "/index-templates/" - + name + ".yaml", version, getVersionProperty()); + final byte[] content = loadVersionedResourceUTF8( + this.getClass(), + "/index-templates/" + name + ".yaml", + version, + getVersionProperty() + ); try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) { return ComposableIndexTemplate.parse(parser); } } catch (Exception e) { - throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); + throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); } } diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 64faaa572a7e0..54323de05c062 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -21,7 +21,6 @@ import java.util.Collections; import java.util.List; - public class OTelPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(OTelPlugin.class); diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java index 83f0164d5ca60..7a248fb0d6349 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java @@ -20,8 +20,14 @@ public class OtelIndexTemplateRegistry extends YamlTemplateRegistry { public static final String OTEL_TEMPLATE_VERSION_VARIABLE = "xpack.oteldata.template.version"; - public OtelIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client, - NamedXContentRegistry xContentRegistry, FeatureService featureService) { + public OtelIndexTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + FeatureService featureService + ) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); } From f0f4e461af051946e10f5a7b6cc47dd29062968d Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Fri, 26 Jul 2024 16:35:52 +0200 Subject: [PATCH 16/51] Update x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml Co-authored-by: Carson Ip --- .../resources/component-templates/metrics-otel@mappings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index 6fc0c7274b97d..eaed6e84c9607 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -8,7 +8,7 @@ template: data_stream.type: type: constant_keyword value: metrics - start_time: + start_timestamp: type: date_nanos metrics: type: passthrough From 595d34456ccd5e11fd4371f7d0061a00cd782178 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 26 Jul 2024 20:29:54 +0200 Subject: [PATCH 17/51] Update 20_metic_tests.yml Workaround for TSDB timestamp issue: we push a custom template with higher priority and set time_series.start_time. --- .../rest-api-spec/test/20_metic_tests.yml | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index 7f06d530e4006..f30573a942ba1 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -1,4 +1,3 @@ - --- setup: - do: @@ -7,8 +6,30 @@ setup: --- "Test push service overview metric": - - skip: - awaits_fix: "TODO: metrics-otel@template uses TSDB, so @timestamp needs to be up-to-date. Q: How to do that in this syntax?" + - do: + indices.put_index_template: + name: test@template + body: + priority: 500 + index_patterns: [ "metrics-*.otel-*" ] + composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - ecs-tsdb@mappings + template: + settings: + index: + routing_path: [unit, attributes.*, resource.attributes.*] + mode: time_series + time_series: + start_time: 2024-07-01T13:03:08.138Z + - do: + indices.get_index_template: + name: metrics-otel@template + - length: {index_templates: 1} - do: bulk: index: metrics-generic.otel-default @@ -21,4 +42,7 @@ setup: - do: search: index: metrics-generic.otel-default + body: + fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ] - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} From 9348218e5d618af6211f8ee3b3c4bf301bcd7c5c Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 26 Jul 2024 20:33:31 +0200 Subject: [PATCH 18/51] Update CODEOWNERS Adding obs-ds-intake-services as owner of the new otel-data plugin. Since we had some changes, also updating the owner of apm-data to the same team. --- .github/CODEOWNERS | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0f7e3073ed022..5b98444c044d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,8 +27,12 @@ libs/logstash-bridge @elastic/logstash x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security # APM Data index templates, etc. -x-pack/plugin/apm-data/src/main/resources @elastic/apm-server -x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/apm-server +x-pack/plugin/apm-data/src/main/resources @elastic/obs-ds-intake-services +x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services + +# OTel +x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services +x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services # Delivery gradle @elastic/es-delivery From 3c680b06fd10a22ab99637e6c61e51c00fd3c3af Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 26 Jul 2024 20:35:36 +0200 Subject: [PATCH 19/51] Change dynamic: strict to false --- .../src/main/resources/component-templates/otel@mappings.yaml | 2 +- .../resources/component-templates/traces-otel@mappings.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index a529dba006bb4..11485c349e2da 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -6,7 +6,7 @@ _meta: template: mappings: date_detection: false - dynamic: strict + dynamic: false properties: "@timestamp": type: date diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 0a4e1c1f2bc9f..6c55a4e38b08d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -6,7 +6,6 @@ _meta: template: mappings: date_detection: false - dynamic: strict properties: data_stream.namespace: type: constant_keyword From e61b441db0637743cc68f08a16cd66c7aea75c8d Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 26 Jul 2024 21:24:23 +0200 Subject: [PATCH 20/51] Skip "Reject invalid top level field" test --- .../main/resources/component-templates/logs-otel@mappings.yaml | 1 - .../resources/rest-api-spec/test/20_traces_tests.yml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index ad505029ea9bf..f4660a4cea5f0 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -6,7 +6,6 @@ _meta: template: mappings: date_detection: false - dynamic: strict properties: data_stream.type: type: constant_keyword diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index ca1f461428f9c..eb3bde4086c4e 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -20,6 +20,8 @@ setup: --- "Reject invalid top level field": + - skip: + awaits_fix: "We decided to allow top level fields for now: https://github.com/elastic/elasticsearch/pull/111091#discussion_r1692988536" - do: bulk: index: traces-generic.otel-default From ecee377c9214c6f2b1fb1ae9d46f97f700fcd95c Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 29 Jul 2024 23:32:46 +0200 Subject: [PATCH 21/51] Update 20_metic_tests.yml --- .../rest-api-spec/test/20_metic_tests.yml | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index f30573a942ba1..7718900feb3e7 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -3,22 +3,10 @@ setup: - do: cluster.health: wait_for_events: languid - ---- -"Test push service overview metric": - do: - indices.put_index_template: - name: test@template + cluster.put_component_template: + name: metrics-otel@custom body: - priority: 500 - index_patterns: [ "metrics-*.otel-*" ] - composed_of: - - metrics@tsdb-settings - - otel@mappings - - metrics@mappings - - metrics-otel@mappings - - semconv-resource-to-ecs@mappings - - ecs-tsdb@mappings template: settings: index: @@ -26,6 +14,8 @@ setup: mode: time_series time_series: start_time: 2024-07-01T13:03:08.138Z +--- +"Test push service overview metric": - do: indices.get_index_template: name: metrics-otel@template @@ -38,7 +28,6 @@ setup: - create: {} - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' - is_false: errors - - do: search: index: metrics-generic.otel-default @@ -46,3 +35,37 @@ setup: fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ] - length: { hits.hits: 1 } - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} +--- +"Query resource attributes as top level": + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - is_false: errors + - do: + search: + index: metrics-generic.otel-default + body: + fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"]} } +--- +"Query attributes as top level": + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - is_false: errors + - do: + search: + index: metrics-generic.otel-default + body: + fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"]} } From 36be6563c4f8d3c2a9fe144b0647ed84a4f82ffb Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 30 Jul 2024 13:48:41 +0200 Subject: [PATCH 22/51] Add boolean as dimension test (skipping it for now) --- .../ecs-tsdb@mappings.yaml | 9 ++++++++- .../rest-api-spec/test/20_metic_tests.yml | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml index 41f380a2a402d..9f263fbb14792 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -1,6 +1,6 @@ version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for the logs index template installed by x-pack + description: Default mappings for the OpenTelemetry tsdb index template installed by x-pack managed: true template: mappings: @@ -10,3 +10,10 @@ template: ignore_above: 1024 type: keyword match_mapping_type: string +# Likely we won't need this, see https://github.com/elastic/elasticsearch/pull/111445 +# Will be very likely removed before merge +# - booleans_to_keywords: +# mapping: +# ignore_above: 1024 +# type: keyword +# match_mapping_type: boolean diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index 7718900feb3e7..cfa2be5e07015 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -69,3 +69,22 @@ setup: fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"] - length: { hits.hits: 1 } - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"]} } +--- +"Boolean as dimension": + - skip: + awaits_fix: "https://github.com/elastic/elasticsearch/pull/111445: Currently a boolean as dimension is rejected" + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}}, "attributes": {"transaction.root":false}}' + - is_false: errors + - do: + search: + index: metrics-generic.otel-default + body: + fields: ["transaction.root"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"transaction.root": [false] }} From 1476c2c7b77c1113edad4251b6e8b57294d34cf1 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 30 Jul 2024 15:43:51 +0200 Subject: [PATCH 23/51] Add booleans_to_keywords and enable corresponding test --- .../component-templates/ecs-tsdb@mappings.yaml | 12 +++++------- .../resources/rest-api-spec/test/20_metic_tests.yml | 4 +--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml index 9f263fbb14792..e3f72457e0fe6 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -10,10 +10,8 @@ template: ignore_above: 1024 type: keyword match_mapping_type: string -# Likely we won't need this, see https://github.com/elastic/elasticsearch/pull/111445 -# Will be very likely removed before merge -# - booleans_to_keywords: -# mapping: -# ignore_above: 1024 -# type: keyword -# match_mapping_type: boolean + - booleans_to_keywords: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: boolean diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index cfa2be5e07015..92d9ec6ddbbb6 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -71,8 +71,6 @@ setup: - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"]} } --- "Boolean as dimension": - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/pull/111445: Currently a boolean as dimension is rejected" - do: bulk: index: metrics-generic.otel-default @@ -87,4 +85,4 @@ setup: body: fields: ["transaction.root"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"transaction.root": [false] }} + - match: { hits.hits.0.fields: {"transaction.root": ["false"] }} From 1487373a9ebf6796fe1db8ab301763b05c8f2dd7 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Wed, 31 Jul 2024 11:33:25 +0200 Subject: [PATCH 24/51] Remove processor.event top level mapping Reason: for metrics and logs we can rely on the name of the datastream. For spans vs. transactions there are other fields we can use. --- .../src/main/resources/component-templates/otel@mappings.yaml | 2 -- .../main/resources/index-templates/traces-otel@template.yaml | 3 --- 2 files changed, 5 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 11485c349e2da..8dc758a89786e 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -23,8 +23,6 @@ template: time_series_dimension: true dropped_attributes_count: type: long - processor.event: - type: keyword scope: properties: name: diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index c508ff9c38c2d..51117fbcddf5f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -8,9 +8,6 @@ _meta: description: default logs template installed by x-pack managed: true composed_of: -# It's TBD how we proceed with these - potentially this could be shared with apm -# - traces@mappings -# - traces@settings - otel@mappings - traces-otel@mappings - semconv-resource-to-ecs@mappings From c47e086ed0c056ed5b805bb20c097a98a04a300c Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Wed, 31 Jul 2024 16:08:58 +0200 Subject: [PATCH 25/51] Remove booleans_to_keywords Because booleans are supported now as dimension on TSDB --- .../resources/component-templates/ecs-tsdb@mappings.yaml | 5 ----- .../resources/rest-api-spec/test/20_metic_tests.yml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml index e3f72457e0fe6..3dcbdbcb1f3ab 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -10,8 +10,3 @@ template: ignore_above: 1024 type: keyword match_mapping_type: string - - booleans_to_keywords: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: boolean diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index 92d9ec6ddbbb6..82343a73f5b71 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -85,4 +85,4 @@ setup: body: fields: ["transaction.root"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"transaction.root": ["false"] }} + - match: { hits.hits.0.fields: {"transaction.root": [false] }} From 42673ce4ae6b49666ffd20c3a8b0f4c0d0b81d1b Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 1 Aug 2024 17:17:56 +0200 Subject: [PATCH 26/51] Add alias service.language.name -> telemetry.sdk.language --- .../semconv-resource-to-ecs@mappings.yaml | 16 +++++++- .../test/30_non_ecs_alias_tests.yml | 37 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml index a32dd42e560a5..c8c972db640b1 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -14,6 +14,9 @@ template: priority: 30 time_series_dimension: true properties: + telemetry.sdk.language: + type: keyword + ignore_above: 1024 service.instance.id: type: keyword ignore_above: 1024 @@ -116,4 +119,15 @@ template: kubernetes.pod.uid: type: alias path: resource.attributes.k8s.pod.uid - +# Below is a non-ECS field, ideally we should have this in a separate file +# I'd imagine having 3 files: +# 1) for the fixed mapping for known resource.attributes.* fields that we want to alias +# 2) this file that aliases ECS -> SemConv +# 3) a new file that aliases non-ECS fields the UI uses -> SemConv +# I implemented it, but always run into such errors: +# [2024-08-01T16:01:49,193][ERROR][o.e.x.c.t.IndexTemplateRegistry] [test-cluster-0] error adding index template [semconv-resource-to-ecs@mappings] for [otel] org.elasticsearch.index.mapper.MapperParsingException: Invalid [path] value [resource.attributes.container.image.tags] for field alias [container.image.tag]: an alias must refer to an existing field in the mappings. +# (That error eventually happens for all fields) - tried changing the order of the 3 files - did not help +# So as a workaround for now, I put this here: + service.language.name: + type: alias + path: resource.attributes.telemetry.sdk.language diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml new file mode 100644 index 0000000000000..975fdac141744 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml @@ -0,0 +1,37 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + - do: + cluster.put_component_template: + name: metrics-otel@custom + body: + template: + settings: + index: + routing_path: [unit, attributes.*, resource.attributes.*] + mode: time_series + time_series: + start_time: 2024-07-01T13:03:08.138Z +--- +"Test alias from service.language.name non-ecs field to telemetry.sdk.language": + - do: + indices.get_index_template: + name: metrics-otel@template + - length: {index_templates: 1} + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - is_false: errors + - do: + search: + index: metrics-generic.otel-default + body: + fields: ["service.language.name"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields: {"service.language.name": ["dotnet"] }} From 140bec0ea4208a45e21877b5b149aad71f5b2b3b Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 5 Aug 2024 19:17:29 +0200 Subject: [PATCH 27/51] cleanup --- ...TemplateRegistry.java => OTelIndexTemplateRegistry.java} | 4 ++-- .../java/org/elasticsearch/xpack/oteldata/OTelPlugin.java | 6 +++--- .../resources/component-templates/logs-otel@mappings.yaml | 2 +- .../component-templates/metrics-otel@mappings.yaml | 2 +- .../main/resources/component-templates/otel@mappings.yaml | 2 +- .../semconv-resource-to-ecs@mappings.yaml | 2 +- .../resources/component-templates/traces-otel@mappings.yaml | 2 +- .../main/resources/component-templates/traces@mappings.yaml | 2 +- .../main/resources/index-templates/logs-otel@template.yaml | 2 +- .../resources/index-templates/metrics-otel@template.yaml | 2 +- .../resources/index-templates/traces-otel@template.yaml | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) rename x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/{OtelIndexTemplateRegistry.java => OTelIndexTemplateRegistry.java} (93%) diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java similarity index 93% rename from x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java rename to x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java index 7a248fb0d6349..34e6dac5a5eca 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OtelIndexTemplateRegistry.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java @@ -16,11 +16,11 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; -public class OtelIndexTemplateRegistry extends YamlTemplateRegistry { +public class OTelIndexTemplateRegistry extends YamlTemplateRegistry { public static final String OTEL_TEMPLATE_VERSION_VARIABLE = "xpack.oteldata.template.version"; - public OtelIndexTemplateRegistry( + public OTelIndexTemplateRegistry( Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 54323de05c062..2beb288b5daf2 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -24,7 +24,7 @@ public class OTelPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(OTelPlugin.class); - final SetOnce registry = new SetOnce<>(); + final SetOnce registry = new SetOnce<>(); private final boolean enabled; @@ -48,7 +48,7 @@ public Collection createComponents(PluginServices services) { Settings settings = services.environment().settings(); ClusterService clusterService = services.clusterService(); registry.set( - new OtelIndexTemplateRegistry( + new OTelIndexTemplateRegistry( settings, clusterService, services.threadPool(), @@ -58,7 +58,7 @@ public Collection createComponents(PluginServices services) { ) ); if (enabled) { - OtelIndexTemplateRegistry registryInstance = registry.get(); + OTelIndexTemplateRegistry registryInstance = registry.get(); registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings)); clusterService.getClusterSettings().addSettingsUpdateConsumer(OTEL_DATA_REGISTRY_ENABLED, registryInstance::setEnabled); registryInstance.initialize(); diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index f4660a4cea5f0..f660647f58054 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for the logs index template installed by x-pack + description: Default mappings for OpenTelemetry logs index template installed by x-pack managed: true template: mappings: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index eaed6e84c9607..ac2e7cab39889 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -1,6 +1,6 @@ version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for the logs index template installed by x-pack + description: Default mappings for the OpenTelemetry metrics index template installed by x-pack managed: true template: mappings: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 8dc758a89786e..a14f0368b94d4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for all OTel data streams + description: Default mappings for all OpenTelemetry data streams managed: true template: mappings: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml index c8c972db640b1..2ec12d0266c77 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for all APM data streams + description: Aliases from OpenTelemetry SemConv fields to ECS (and some non-ECS) fields managed: true template: mappings: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 6c55a4e38b08d..375e9d2e77c8f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for all APM data streams + description: Default mappings for OpenTelemetry traces managed: true template: mappings: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml index 1e06cfa58a25e..5617e40f524a5 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.oteldata.template.version} _meta: - description: Default mappings for the traces index template installed by x-pack + description: Default mappings for traces index template installed by x-pack managed: true template: mappings: diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml index dbb29e73ce656..2151801bce674 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml @@ -5,7 +5,7 @@ priority: 120 data_stream: {} allow_auto_create: true _meta: - description: default logs template installed by x-pack + description: default OpenTelemetry logs template installed by x-pack managed: true composed_of: - logs@mappings diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index 2926b36fd0432..b3a0da4003607 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -5,7 +5,7 @@ priority: 120 data_stream: {} allow_auto_create: true _meta: - description: default logs template installed by x-pack + description: default OpenTelemetry metrics template installed by x-pack managed: true composed_of: - metrics@tsdb-settings diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index 51117fbcddf5f..1307dc4c3a432 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -5,7 +5,7 @@ priority: 120 data_stream: {} allow_auto_create: true _meta: - description: default logs template installed by x-pack + description: default OpenTelemetry traces template installed by x-pack managed: true composed_of: - otel@mappings From 1c6192990883db59473710541e08489c679f33b2 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 5 Aug 2024 19:31:17 +0200 Subject: [PATCH 28/51] Update README.md --- x-pack/plugin/otel-data/README.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/x-pack/plugin/otel-data/README.md b/x-pack/plugin/otel-data/README.md index 862ec7cbb025b..c32ba684fe5e8 100644 --- a/x-pack/plugin/otel-data/README.md +++ b/x-pack/plugin/otel-data/README.md @@ -1,5 +1,3 @@ -NOTE: this plugin is not related to APM Metrics used in ES codebase. The APM Metrics are in :modules:apm - ## OpenTelemetry Ingest plugin The OpenTelemetry Ingest plugin installs index templates and component templates for Elastic OpenTelemetry data. @@ -22,22 +20,13 @@ Any update to resources included by this package also requires a bump to the ## Testing -## Unit testing - -Java unit tests cover basic, low-level details of the plugin, such as the parsing and loading of resources. -These can be run with: - -``` -./gradlew x-pack:plugin:apm-data:test -``` - ## Integration testing The index templates and ingest pipeline functionality is tested using YAML REST tests. These can be run with: ``` -./gradlew x-pack:plugin:apm-data:yamlRestTest +./gradlew x-pack:plugin:otel-data:yamlRestTest ``` Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc) From 63690edf645d49b39f0ded605ed23daaeca6cda6 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 5 Aug 2024 19:38:31 +0200 Subject: [PATCH 29/51] Update README.md --- x-pack/plugin/otel-data/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/README.md b/x-pack/plugin/otel-data/README.md index c32ba684fe5e8..214990c2d352f 100644 --- a/x-pack/plugin/otel-data/README.md +++ b/x-pack/plugin/otel-data/README.md @@ -26,7 +26,7 @@ The index templates and ingest pipeline functionality is tested using YAML REST These can be run with: ``` -./gradlew x-pack:plugin:otel-data:yamlRestTest +./gradlew :x-pack:plugin:otel-data:yamlRestTest ``` Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc) From c9b617f192394e74a7750092592cd5ab836bf021 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Tue, 6 Aug 2024 11:03:47 +0200 Subject: [PATCH 30/51] Update docs/changelog/111091.yaml --- docs/changelog/111091.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/111091.yaml diff --git a/docs/changelog/111091.yaml b/docs/changelog/111091.yaml new file mode 100644 index 0000000000000..8444681a14a48 --- /dev/null +++ b/docs/changelog/111091.yaml @@ -0,0 +1,5 @@ +pr: 111091 +summary: "X-pack/plugin/otel: introduce x-pack-otel plugin" +area: Data streams +type: feature +issues: [] From f42756445d4e409b12d23a26ed85fa3b7a6508a0 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 6 Aug 2024 19:20:58 +0200 Subject: [PATCH 31/51] Move traces@settings and traces@mappings to core --- docs/changelog/111091.yaml | 3 +- .../component-templates/traces@mappings.yaml | 11 -------- .../src/main/resources/resources.yaml | 1 - .../src/main/resources/traces@mappings.json | 28 +++++++++++++++++++ .../src/main/resources/traces@settings.json | 21 ++++++++++++++ .../component-templates/traces@mappings.yaml | 17 ----------- .../component-templates/traces@settings.yaml | 12 -------- .../index-templates/traces-otel@template.yaml | 2 ++ .../xpack/stack/StackTemplateRegistry.java | 22 ++++++++++++++- .../stack/StackTemplateRegistryTests.java | 14 ++++++++-- 10 files changed, 86 insertions(+), 45 deletions(-) delete mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json delete mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml delete mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml diff --git a/docs/changelog/111091.yaml b/docs/changelog/111091.yaml index 8444681a14a48..8f4675a60ff2a 100644 --- a/docs/changelog/111091.yaml +++ b/docs/changelog/111091.yaml @@ -2,4 +2,5 @@ pr: 111091 summary: "X-pack/plugin/otel: introduce x-pack-otel plugin" area: Data streams type: feature -issues: [] +issues: + - 104731 diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml deleted file mode 100644 index 51c987df4df60..0000000000000 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -version: ${xpack.apmdata.template.version} -_meta: - description: Default mappings for traces data streams - managed: true -template: - mappings: - properties: - data_stream.type: - type: constant_keyword - value: traces diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index efa6ae694c464..fa38fda679e49 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -23,7 +23,6 @@ component-templates: - metrics-apm.service_summary@mappings - metrics-apm.service_transaction@mappings - metrics-apm.transaction@mappings - - traces@mappings - traces-apm@mappings - traces-apm.rum@mappings diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json new file mode 100644 index 0000000000000..e3990a250f0c2 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json @@ -0,0 +1,28 @@ +{ + "template": { + "mappings": { + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "data_stream.type": { + "type": "constant_keyword", + "value": "traces" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + }, + "_meta": { + "description": "default mappings for the traces index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json new file mode 100644 index 0000000000000..b9774bc04d2aa --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json @@ -0,0 +1,21 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "traces" + }, + "codec": "best_compression", + "mapping": { + "ignore_malformed": true + } + } + } + }, + "_meta": { + "description": "default settings for the traces index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml deleted file mode 100644 index 5617e40f524a5..0000000000000 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@mappings.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -version: ${xpack.oteldata.template.version} -_meta: - description: Default mappings for traces index template installed by x-pack - managed: true -template: - mappings: - properties: - "@timestamp": - type: date - data_stream.type: - type: constant_keyword - value: traces - data_stream.dataset: - type: constant_keyword - data_stream.namespace: - type: constant_keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml deleted file mode 100644 index 362f3110b7838..0000000000000 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces@settings.yaml +++ /dev/null @@ -1,12 +0,0 @@ -version: ${xpack.oteldata.template.version} -_meta: - description: Default settings for OpenTelemetry traces - managed: true -template: - settings: -# index: -# lifecycle: -# name: traces - codec: best_compression - mapping: - ignore_malformed: true diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index 1307dc4c3a432..03f69d31db3cb 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -8,6 +8,8 @@ _meta: description: default OpenTelemetry traces template installed by x-pack managed: true composed_of: + - traces@mappings + - traces@settings - otel@mappings - traces-otel@mappings - semconv-resource-to-ecs@mappings diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 6a9936f4f27d3..9e847455d2c86 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -48,7 +48,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 12; + public static final int REGISTRY_VERSION = 13; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( @@ -106,6 +106,12 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { public static final String METRICS_ILM_POLICY_NAME = "metrics@lifecycle"; public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics"; + ////////////////////////////////////////////////////////// + // Base traces components + ////////////////////////////////////////////////////////// + public static final String TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME = "traces@mappings"; + public static final String TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME = "traces@settings"; + ////////////////////////////////////////////////////////// // Synthetics components (for matching synthetics-*-* indices) ////////////////////////////////////////////////////////// @@ -192,6 +198,20 @@ private Map loadComponentTemplateConfigs(boolean logs TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES ), + new IndexTemplateConfig( + TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/traces@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/traces@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), new IndexTemplateConfig( SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, "/synthetics@mappings.json", diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java index abb2d5765b128..25ff3b5311fa2 100644 --- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -430,10 +430,12 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes); client.setVerifier((action, request, listener) -> { - if (action instanceof PutComponentTemplateAction) { - fail("template should not have been re-installed"); + if (request instanceof PutComponentTemplateAction.Request put) { + fail("template should not have been re-installed: " + put.name()); return null; } else if (action == ILMActions.PUT) { // Ignore this, it's verified in another test @@ -489,6 +491,14 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) ); + versions.put( + StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes); registry.clusterChanged(higherVersionEvent); } From 7e061318d5bb51b8529496be98e3013188d81aa2 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 6 Aug 2024 19:27:44 +0200 Subject: [PATCH 32/51] Update traces-otel@mappings.yaml --- .../resources/component-templates/traces-otel@mappings.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 375e9d2e77c8f..b193bd11d9baf 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -7,9 +7,6 @@ template: mappings: date_detection: false properties: - data_stream.namespace: - type: constant_keyword - value: traces trace_id: type: keyword trace.id: From 717a29aed01c56c2dde8b44156fb43f3411ea4d1 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Wed, 7 Aug 2024 17:06:10 +0200 Subject: [PATCH 33/51] Review feedback --- .../component-templates/otel@mappings.yaml | 2 ++ .../semconv-resource-to-ecs@mappings.yaml | 10 +--------- .../index-templates/metrics-otel@template.yaml | 2 +- .../rest-api-spec/test/20_metic_tests.yml | 14 ++++++++------ .../rest-api-spec/test/20_traces_tests.yml | 16 ---------------- .../test/30_non_ecs_alias_tests.yml | 2 +- 6 files changed, 13 insertions(+), 33 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index a14f0368b94d4..486b9b7a70ade 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -7,6 +7,8 @@ template: mappings: date_detection: false dynamic: false + _source: + mode: synthetic properties: "@timestamp": type: date diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml index 2ec12d0266c77..50443c1adb1c7 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -119,15 +119,7 @@ template: kubernetes.pod.uid: type: alias path: resource.attributes.k8s.pod.uid -# Below is a non-ECS field, ideally we should have this in a separate file -# I'd imagine having 3 files: -# 1) for the fixed mapping for known resource.attributes.* fields that we want to alias -# 2) this file that aliases ECS -> SemConv -# 3) a new file that aliases non-ECS fields the UI uses -> SemConv -# I implemented it, but always run into such errors: -# [2024-08-01T16:01:49,193][ERROR][o.e.x.c.t.IndexTemplateRegistry] [test-cluster-0] error adding index template [semconv-resource-to-ecs@mappings] for [otel] org.elasticsearch.index.mapper.MapperParsingException: Invalid [path] value [resource.attributes.container.image.tags] for field alias [container.image.tag]: an alias must refer to an existing field in the mappings. -# (That error eventually happens for all fields) - tried changing the order of the 3 files - did not help -# So as a workaround for now, I put this here: +# Below are non-ECS fields that may be used by Kibana. service.language.name: type: alias path: resource.attributes.telemetry.sdk.language diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index b3a0da4003607..578fc7ff1a291 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -8,9 +8,9 @@ _meta: description: default OpenTelemetry metrics template installed by x-pack managed: true composed_of: - - metrics@tsdb-settings - otel@mappings - metrics@mappings + - metrics@tsdb-settings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index 82343a73f5b71..4ce81fa640b05 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -26,7 +26,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0},"attributes":{"processor.event":"metric"}}' - is_false: errors - do: search: @@ -51,7 +51,9 @@ setup: body: fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"]} } + - match: { hits.hits.0.fields: { "service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} +# - match: { hits.hits.0.fields.telemetry.sdk.language: ["dotnet"] } +# - match: { hits.hits.0.fields.telemetry.sdk.name: ["opentelemetry"] } --- "Query attributes as top level": - do: @@ -60,15 +62,15 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{"processor.event":"metric", "foo": "bar"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' - is_false: errors - do: search: index: metrics-generic.otel-default body: - fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"] + fields: ["foo"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"]} } + - match: { hits.hits.0.fields: {"foo": ["bar"] }} --- "Boolean as dimension": - do: @@ -77,7 +79,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}}, "attributes": {"transaction.root":false}}' + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{"processor.event":"metric","transaction.root":false},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}}}' - is_false: errors - do: search: diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index eb3bde4086c4e..5992f604d7607 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -19,22 +19,6 @@ setup: - length: { hits.hits: 1 } --- -"Reject invalid top level field": - - skip: - awaits_fix: "We decided to allow top level fields for now: https://github.com/elastic/elasticsearch/pull/111091#discussion_r1692988536" - - do: - bulk: - index: traces-generic.otel-default - refresh: true - body: - - create: {} - - '{"invalid_top_level_field": "foo", "@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - - is_true: errors - - do: - search: - index: traces-generic.otel-default - - length: { hits.hits: 0 } ---- "Query resource attributes as top level": - do: bulk: diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml index 975fdac141744..58ef47a2dd1d9 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml @@ -26,7 +26,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"}, "attributes": {"processor.event":"metric"}, "resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}' - is_false: errors - do: search: From d5de2a1625866f227c84efd3c44b198dc2659a65 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Wed, 7 Aug 2024 17:41:11 +0200 Subject: [PATCH 34/51] Adapt `match` style in tests --- .../rest-api-spec/test/20_metic_tests.yml | 14 ++++++++------ .../rest-api-spec/test/20_traces_tests.yml | 9 ++++++--- .../rest-api-spec/test/30_non_ecs_alias_tests.yml | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml index 4ce81fa640b05..25eb4831836f2 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml @@ -34,7 +34,9 @@ setup: body: fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} + - match: { hits.hits.0.fields.service\.name: [ "OtelSample" ] } + - match: { hits.hits.0.fields.telemetry\.sdk\.language: [ "dotnet" ] } + - match: { hits.hits.0.fields.telemetry\.sdk\.name: [ "opentelemetry" ] } --- "Query resource attributes as top level": - do: @@ -51,9 +53,9 @@ setup: body: fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: { "service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} -# - match: { hits.hits.0.fields.telemetry.sdk.language: ["dotnet"] } -# - match: { hits.hits.0.fields.telemetry.sdk.name: ["opentelemetry"] } + - match: { hits.hits.0.fields.service\.name: [ "OtelSample" ] } + - match: { hits.hits.0.fields.telemetry\.sdk\.language: [ "dotnet" ] } + - match: { hits.hits.0.fields.telemetry\.sdk\.name: [ "opentelemetry" ] } --- "Query attributes as top level": - do: @@ -70,7 +72,7 @@ setup: body: fields: ["foo"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"foo": ["bar"] }} + - match: { hits.hits.0.fields.foo\: ["bar"] } --- "Boolean as dimension": - do: @@ -87,4 +89,4 @@ setup: body: fields: ["transaction.root"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"transaction.root": [false] }} + - match: { hits.hits.0.fields.transaction\.root: [false] } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index 5992f604d7607..f1ddcbcf38dcb 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -34,8 +34,9 @@ setup: body: fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"service.name": ["OtelSample"], "telemetry.sdk.language": ["dotnet"], "telemetry.sdk.name": ["opentelemetry"] }} - + - match: { hits.hits.0.fields.service\.name: [ "OtelSample" ] } + - match: { hits.hits.0.fields.telemetry\.sdk\.language: [ "dotnet" ] } + - match: { hits.hits.0.fields.telemetry\.sdk\.name: [ "opentelemetry" ] } --- "Query attributes as top level": - do: @@ -52,4 +53,6 @@ setup: body: fields: ["db.type", "db.name", "db.operation", "db.statement"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"db.type": ["mssql"], "db.name": ["foo"], "db.operation": ["SELECT"], "db.statement": ["SELECT * FROM wuser_table"] }} + - match: { hits.hits.0.fields.db\.type: [ "mssql" ] } + - match: { hits.hits.0.fields.db\.operation: [ "SELECT" ] } + - match: { hits.hits.0.fields.db\.statement: [ "SELECT * FROM wuser_table" ] } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml index 58ef47a2dd1d9..d80c52c756b54 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml @@ -34,4 +34,4 @@ setup: body: fields: ["service.language.name"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields: {"service.language.name": ["dotnet"] }} + - match: { hits.hits.0.fields.service\.language\.name: [ "dotnet" ] } From 4ca5a21f18c3074a76f839e57bd8ba8ae2d4dd98 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Wed, 7 Aug 2024 18:14:12 +0200 Subject: [PATCH 35/51] Update docs/changelog/111091.yaml --- docs/changelog/111091.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/111091.yaml b/docs/changelog/111091.yaml index 8f4675a60ff2a..8444681a14a48 100644 --- a/docs/changelog/111091.yaml +++ b/docs/changelog/111091.yaml @@ -2,5 +2,4 @@ pr: 111091 summary: "X-pack/plugin/otel: introduce x-pack-otel plugin" area: Data streams type: feature -issues: - - 104731 +issues: [] From 65bfd0b17d1c3694f146b69cd00ec6ea2e2cb7c2 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Wed, 7 Aug 2024 19:09:30 +0200 Subject: [PATCH 36/51] Apply suggestions from code review Co-authored-by: Vishal Raj --- .../component-templates/semconv-resource-to-ecs@mappings.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml index 50443c1adb1c7..f72ac9eeb22af 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -35,7 +35,7 @@ template: process.executable.path: type: keyword ignore_above: 1024 - process.runtime.path: + process.runtime.name: type: keyword ignore_above: 1024 process.runtime.version: @@ -88,7 +88,7 @@ template: path: resource.attributes.process.executable.path service.runtime.name: type: alias - path: resource.attributes.process.runtime.path + path: resource.attributes.process.runtime.name service.runtime.version: type: alias path: resource.attributes.process.runtime.version From 10287a72fcfff877b6dff6ec386aca44bc35d265 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Wed, 7 Aug 2024 19:12:18 +0200 Subject: [PATCH 37/51] Update x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml Co-authored-by: Carson Ip --- .../resources/component-templates/traces-otel@mappings.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index b193bd11d9baf..559f1174a3743 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -26,8 +26,6 @@ template: path: parent_span_id trace_flags: type: byte - parent_is_remote: - type: boolean name: type: keyword span.name: From a6baf4644924ea4929ae2ab58ebac0b78f936e62 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Thu, 8 Aug 2024 18:25:54 +0200 Subject: [PATCH 38/51] Changing trace_flags to long Related discussion: https://github.com/elastic/elasticsearch/pull/111091#discussion_r1706698491 --- .../logs-otel@mappings.yaml | 2 +- .../traces-otel@mappings.yaml | 4 ++-- .../rest-api-spec/test/20_traces_tests.yml | 20 +++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index f660647f58054..751d8d3dc64ca 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -27,7 +27,7 @@ template: body_structured: type: flattened trace_flags: - type: byte + type: long trace_id: type: keyword trace.id: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 559f1174a3743..387a7bb881097 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -25,7 +25,7 @@ template: type: alias path: parent_span_id trace_flags: - type: byte + type: long name: type: keyword span.name: @@ -54,7 +54,7 @@ template: dropped_attributes_count: type: long trace_flags: - type: byte + type: long dropped_links_count: type: long status: diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index f1ddcbcf38dcb..7922ac8c39845 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -56,3 +56,23 @@ setup: - match: { hits.hits.0.fields.db\.type: [ "mssql" ] } - match: { hits.hits.0.fields.db\.operation: [ "SELECT" ] } - match: { hits.hits.0.fields.db\.statement: [ "SELECT * FROM wuser_table" ] } +--- +"Sending non-zeroed trace flags": +# Undefined flags in trace flags are not guaranteed to be 0. +# According to comments in the collector code, readers should not expect those to be all 0. +# Therefore, this test sends an all 1's flag (max-int: 4294967295) and makes sure it's not rejected. + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z", "trace_flags": 4294967295, "data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - is_false: errors + - do: + search: + index: traces-generic.otel-default + body: + fields: ["trace_flags",] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.trace_flags\: [ 4294967295 ] } From 7989b123da7bbfc3648b8516b07725f3c9d13e2f Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 9 Aug 2024 16:18:57 +0200 Subject: [PATCH 39/51] Remove trace_flags see: https://github.com/elastic/opentelemetry-dev/pull/368#pullrequestreview-2229633970 --- .../logs-otel@mappings.yaml | 2 -- .../traces-otel@mappings.yaml | 4 ---- .../rest-api-spec/test/20_traces_tests.yml | 21 +------------------ 3 files changed, 1 insertion(+), 26 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index 751d8d3dc64ca..618339a61b5fa 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -26,8 +26,6 @@ template: path: body_text body_structured: type: flattened - trace_flags: - type: long trace_id: type: keyword trace.id: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 387a7bb881097..f62a7d288245c 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -24,8 +24,6 @@ template: parent.id: type: alias path: parent_span_id - trace_flags: - type: long name: type: keyword span.name: @@ -53,8 +51,6 @@ template: dynamic: true dropped_attributes_count: type: long - trace_flags: - type: long dropped_links_count: type: long status: diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index 7922ac8c39845..6e64825527999 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -56,23 +56,4 @@ setup: - match: { hits.hits.0.fields.db\.type: [ "mssql" ] } - match: { hits.hits.0.fields.db\.operation: [ "SELECT" ] } - match: { hits.hits.0.fields.db\.statement: [ "SELECT * FROM wuser_table" ] } ---- -"Sending non-zeroed trace flags": -# Undefined flags in trace flags are not guaranteed to be 0. -# According to comments in the collector code, readers should not expect those to be all 0. -# Therefore, this test sends an all 1's flag (max-int: 4294967295) and makes sure it's not rejected. - - do: - bulk: - index: traces-generic.otel-default - refresh: true - body: - - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z", "trace_flags": 4294967295, "data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - - is_false: errors - - do: - search: - index: traces-generic.otel-default - body: - fields: ["trace_flags",] - - length: { hits.hits: 1 } - - match: { hits.hits.0.fields.trace_flags\: [ 4294967295 ] } + From 2a44ecd12aa214a4bdbfd74a0aaa869b799b6e06 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Fri, 9 Aug 2024 16:45:19 +0200 Subject: [PATCH 40/51] Apply suggestions from code review Co-authored-by: Andrew Wilkins --- x-pack/plugin/otel-data/README.md | 2 +- .../main/resources/index-templates/traces-otel@template.yaml | 1 - x-pack/plugin/otel-data/src/main/resources/resources.yaml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/otel-data/README.md b/x-pack/plugin/otel-data/README.md index 214990c2d352f..7cab6bfa453d8 100644 --- a/x-pack/plugin/otel-data/README.md +++ b/x-pack/plugin/otel-data/README.md @@ -1,6 +1,6 @@ ## OpenTelemetry Ingest plugin -The OpenTelemetry Ingest plugin installs index templates and component templates for Elastic OpenTelemetry data. +The OpenTelemetry Ingest plugin installs index templates and component templates for OpenTelemetry data. All resources are defined as YAML under [src/main/resources](src/main/resources). diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index 03f69d31db3cb..873ca59c15ec3 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -19,4 +19,3 @@ composed_of: ignore_missing_component_templates: - traces@custom - traces-otel@custom - diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index 8760b5922d3db..c887c32316530 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,6 +1,6 @@ # "version" holds the version of the templates and ingest pipelines installed -# by xpack-plugin otel-data. This must be increased whenever an existing template or -# pipeline is changed, in order for it to be updated on Elasticsearch upgrade. +# by xpack-plugin otel-data. This must be increased whenever an existing template is +# changed, in order for it to be updated on Elasticsearch upgrade. version: 1 component-templates: From a3837fc60c8d6f6046a2e064053b9eae8acb52d3 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Fri, 9 Aug 2024 20:06:13 +0200 Subject: [PATCH 41/51] Review feedback --- .../java/org/elasticsearch/xpack/oteldata/OTelPlugin.java | 2 ++ .../resources/component-templates/logs-otel@mappings.yaml | 1 - .../main/resources/component-templates/otel@mappings.yaml | 1 + .../component-templates/traces-otel@mappings.yaml | 1 - .../otel-data/src/main/resources/index-templates/README.md | 7 ------- .../resources/index-templates/metrics-otel@template.yaml | 2 +- .../test/{20_metic_tests.yml => 20_metrics_tests.yml} | 0 7 files changed, 4 insertions(+), 10 deletions(-) delete mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/README.md rename x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/{20_metic_tests.yml => 20_metrics_tests.yml} (100%) diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 2beb288b5daf2..f2ff378ab8483 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -33,6 +33,8 @@ public class OTelPlugin extends Plugin implements ActionPlugin { // This setting will be ignored if the plugin is disabled. static final Setting OTEL_DATA_REGISTRY_ENABLED = Setting.boolSetting( "xpack.otel_data.registry.enabled", + // OTel-data is under development, and we start with opt-in first. + // Furthermore, this could help with staged rollout in serverless false, Setting.Property.NodeScope, Setting.Property.Dynamic diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index 618339a61b5fa..87f0e517f9fef 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -5,7 +5,6 @@ _meta: managed: true template: mappings: - date_detection: false properties: data_stream.type: type: constant_keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 486b9b7a70ade..3998343ea99f0 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -11,6 +11,7 @@ template: mode: synthetic properties: "@timestamp": +#Ultimeately we aim to use date_nanos. Waiting for https://github.com/elastic/elasticsearch/issues/109352 type: date data_stream.type: type: constant_keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index f62a7d288245c..b9eca7809bf34 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -5,7 +5,6 @@ _meta: managed: true template: mappings: - date_detection: false properties: trace_id: type: keyword diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/README.md b/x-pack/plugin/otel-data/src/main/resources/index-templates/README.md deleted file mode 100644 index 99f4f4218ca90..0000000000000 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Mappings and settings that must not be overridden are to be defined -in the index templates, as these will take precedence over component -template mappings and settings. This includes: - - - `index.default_pipeline` - - `index.final_pipeline` - - values for `data_stream.*` diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index 578fc7ff1a291..e718dd01b6c61 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -8,9 +8,9 @@ _meta: description: default OpenTelemetry metrics template installed by x-pack managed: true composed_of: - - otel@mappings - metrics@mappings - metrics@tsdb-settings + - otel@mappings - metrics-otel@mappings - semconv-resource-to-ecs@mappings - metrics@custom diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml similarity index 100% rename from x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metic_tests.yml rename to x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml From 846db1d13759a5d4edff0ec8647f81969bc54d07 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 12 Aug 2024 19:07:25 +0200 Subject: [PATCH 42/51] Add store_array_source for span links --- .../traces-otel@mappings.yaml | 3 +++ .../rest-api-spec/test/20_traces_tests.yml | 20 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index b9eca7809bf34..8a2596e6cf79f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -5,6 +5,8 @@ _meta: managed: true template: mappings: + _source: + mode: synthetic properties: trace_id: type: keyword @@ -37,6 +39,7 @@ template: dropped_events_count: type: long links: + store_array_source: true properties: trace_id: type: keyword diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index 6e64825527999..c597fac6cf7f6 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -56,4 +56,22 @@ setup: - match: { hits.hits.0.fields.db\.type: [ "mssql" ] } - match: { hits.hits.0.fields.db\.operation: [ "SELECT" ] } - match: { hits.hits.0.fields.db\.statement: [ "SELECT * FROM wuser_table" ] } - +--- +"Span links test": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"links":[{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"086e83747d0e381e","attributes":{"foo":"bar"}},{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"b3b7d1f1f1b4e1e1"}],"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - is_false: errors + - do: + search: + index: traces-generic.otel-default + - length: { hits.hits.0._source.links: 2 } + - match: { hits.hits.0._source.links.0.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1" } + - match: { hits.hits.0._source.links.0.span_id: "086e83747d0e381e" } + - match: { hits.hits.0._source.links.0.attributes.foo: "bar" } + - match: { hits.hits.0._source.links.1.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1" } + - match: { hits.hits.0._source.links.1.span_id: "b3b7d1f1f1b4e1e1" } From e1adb6ac8acc3481ecc27ce120cb553fa13554a4 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 12 Aug 2024 20:15:31 +0200 Subject: [PATCH 43/51] Define constant `data_stream.type` in `template.yaml`s --- .../metrics-otel@mappings.yaml | 3 -- .../index-templates/logs-otel@template.yaml | 6 ++++ .../metrics-otel@template.yaml | 5 +++ .../index-templates/traces-otel@template.yaml | 6 ++++ .../rest-api-spec/test/20_logs.tests.yml | 33 +++++++++++++++++++ .../rest-api-spec/test/20_metrics_tests.yml | 17 ++++++++++ .../rest-api-spec/test/20_traces_tests.yml | 25 +++++++++++--- 7 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml index ac2e7cab39889..b7a17dba973f8 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml @@ -5,9 +5,6 @@ _meta: template: mappings: properties: - data_stream.type: - type: constant_keyword - value: metrics start_timestamp: type: date_nanos metrics: diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml index 2151801bce674..6772ec5bc65d4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml @@ -19,3 +19,9 @@ composed_of: ignore_missing_component_templates: - logs@custom - logs-otel@custom +template: + mappings: + properties: + data_stream.type: + type: constant_keyword + value: logs diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index e718dd01b6c61..53442b544bb6a 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -23,3 +23,8 @@ template: settings: index: mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml index 873ca59c15ec3..370b9351c16f5 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml @@ -19,3 +19,9 @@ composed_of: ignore_missing_component_templates: - traces@custom - traces-otel@custom +template: + mappings: + properties: + data_stream.type: + type: constant_keyword + value: traces diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml new file mode 100644 index 0000000000000..d4dea23022121 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml @@ -0,0 +1,33 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + - do: + cluster.put_component_template: + name: logs-otel@custom + body: + template: + settings: + index: + routing_path: [unit, attributes.*, resource.attributes.*] + mode: time_series + time_series: + start_time: 2024-07-01T13:03:08.138Z +--- +"Default data_stream.type must be logs": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body_text":"Error: Unable to connect to the database.","severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}' + - is_false: errors + - do: + search: + index: logs-generic.otel-default + body: + fields: ["data_stream.type"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.data_stream\.type: ["logs"] } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index 25eb4831836f2..730115142464f 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -90,3 +90,20 @@ setup: fields: ["transaction.root"] - length: { hits.hits: 1 } - match: { hits.hits.0.fields.transaction\.root: [false] } +--- +"Default data_stream.type must be metrics": + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"},"attributes":{"processor.event":"metric","transaction.root":false},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}}}' + - is_false: errors + - do: + search: + index: metrics-generic.otel-default + body: + fields: ["data_stream.type"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.data_stream\.type: ["metrics"] } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index c597fac6cf7f6..abdb8d49d774c 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -11,7 +11,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: @@ -26,7 +26,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: @@ -45,7 +45,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: @@ -64,7 +64,7 @@ setup: refresh: true body: - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"traces","type":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"links":[{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"086e83747d0e381e","attributes":{"foo":"bar"}},{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"b3b7d1f1f1b4e1e1"}],"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"links":[{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"086e83747d0e381e","attributes":{"foo":"bar"}},{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"b3b7d1f1f1b4e1e1"}],"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - is_false: errors - do: search: @@ -75,3 +75,20 @@ setup: - match: { hits.hits.0._source.links.0.attributes.foo: "bar" } - match: { hits.hits.0._source.links.1.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1" } - match: { hits.hits.0._source.links.1.span_id: "b3b7d1f1f1b4e1e1" } +--- +"Default data_stream.type must be traces": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' + - is_false: errors + - do: + search: + index: traces-generic.otel-default + body: + fields: ["data_stream.type"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.data_stream\.type: ["traces"] } From dfdcf709259789e33e236f69006448828afa73db Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Mon, 12 Aug 2024 21:42:24 +0200 Subject: [PATCH 44/51] Create package-info.java --- .../xpack/oteldata/package-info.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java new file mode 100644 index 0000000000000..98c6c9a3999c4 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * This package contains index templates for OpenTelemetry data. It covers traces (spans), metrics, and logs. + * The plugin is expected to be used in combination with the Elasticsearch exporter defined as the exporter + * within an OpenTelemetry collector with the mapping mode `otel`. + * For more information about the Elasticsearch exporter + * @see + * https://github.com/open-telemetry/opentelemetry-collector-contrib. + * + */ +package org.elasticsearch.xpack.oteldata; From 16234d7671d1e9785c8f15d177ed632b535a08b5 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 13 Aug 2024 11:47:55 +0200 Subject: [PATCH 45/51] Move ecs-tsdb@mappings to index template Add test to verify that @custom template can add dynamic templates with a higher precedence --- .../ecs-tsdb@mappings.yaml | 12 ------ .../metrics-otel@template.yaml | 8 +++- .../src/main/resources/resources.yaml | 1 - .../rest-api-spec/test/20_metrics_tests.yml | 40 +++++++++++++++++++ 4 files changed, 47 insertions(+), 14 deletions(-) delete mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml deleted file mode 100644 index 3dcbdbcb1f3ab..0000000000000 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml +++ /dev/null @@ -1,12 +0,0 @@ -version: ${xpack.oteldata.template.version} -_meta: - description: Default mappings for the OpenTelemetry tsdb index template installed by x-pack - managed: true -template: - mappings: - dynamic_templates: - - all_strings_to_keywords: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index 53442b544bb6a..89ff28249aabb 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -15,7 +15,6 @@ composed_of: - semconv-resource-to-ecs@mappings - metrics@custom - metrics-otel@custom - - ecs-tsdb@mappings ignore_missing_component_templates: - metrics@custom - metrics-otel@custom @@ -28,3 +27,10 @@ template: data_stream.type: type: constant_keyword value: metrics + dynamic_templates: + - all_strings_to_keywords: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index c887c32316530..8e0a7606cbd05 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -7,7 +7,6 @@ component-templates: - otel@mappings - logs-otel@mappings - semconv-resource-to-ecs@mappings - - ecs-tsdb@mappings - metrics-otel@mappings - traces-otel@mappings index-templates: diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index 730115142464f..a6591d6c32210 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -107,3 +107,43 @@ setup: fields: ["data_stream.type"] - length: { hits.hits: 1 } - match: { hits.hits.0.fields.data_stream\.type: ["metrics"] } + +--- +"Custom dynamic template": + - do: + cluster.put_component_template: + name: metrics-otel@custom + body: + template: + settings: + index: + routing_path: [unit, attributes.*, resource.attributes.*] + mode: time_series + time_series: + start_time: 2024-07-01T13:03:08.138Z + mappings: + dynamic_templates: + - ip_fields: + mapping: + type: ip + match_mapping_type: string + path_match: "*.ip" + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"},"attributes":{"host.ip":"127.0.0.1","foo":"bar"}}' + - is_false: errors + - do: + indices.get_data_stream: + name: metrics-generic.otel-default + - set: { data_streams.0.indices.0.index_name: idx0name } + + - do: + indices.get_mapping: + index: $idx0name + expand_wildcards: hidden + - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } + - match: { .$idx0name.mappings.properties.attributes.properties.foo.type: "keyword" } From 0389c3716f515d24e3dc8452990fcd310133040a Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 13 Aug 2024 16:45:28 +0200 Subject: [PATCH 46/51] Update metrics@mappings.json Remove summary_gauge and summary_counter since they are covered by summary_metrics --- .../src/main/resources/metrics@mappings.json | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json index d4ddbf8f31de2..9c58322f12d03 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json @@ -87,24 +87,6 @@ "ignore_malformed": true } } - }, - { - "summary_gauge": { - "mapping": { - "type": "aggregate_metric_double", - "time_series_metric": "gauge", - "ignore_malformed": true - } - } - }, - { - "summary_counter": { - "mapping": { - "type": "aggregate_metric_double", - "time_series_metric": "counter", - "ignore_malformed": true - } - } } ], "properties": { From 9b93b75d07f83f77d8b17b8f556456a7aa17789e Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 13 Aug 2024 17:59:44 +0200 Subject: [PATCH 47/51] Move clusterService.getClusterSettings().addSettingsUpdateConsumer to registry --- .../xpack/apmdata/APMIndexTemplateRegistry.java | 10 ++++++++++ .../org/elasticsearch/xpack/apmdata/APMPlugin.java | 1 - .../xpack/oteldata/OTelIndexTemplateRegistry.java | 10 ++++++++++ .../org/elasticsearch/xpack/oteldata/OTelPlugin.java | 1 - 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 10c02fd7f1036..40edd53a789ec 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -16,6 +16,8 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; +import static org.elasticsearch.xpack.apmdata.APMPlugin.APM_DATA_REGISTRY_ENABLED; + /** * Creates all index templates and ingest pipelines that are required for using Elastic APM. */ @@ -39,6 +41,14 @@ public String getName() { return "apm"; } + @Override + public void initialize() { + super.initialize(); + if(isEnabled()){ + clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_DATA_REGISTRY_ENABLED, this::setEnabled); + } + } + @Override protected String getVersionProperty() { return APM_TEMPLATE_VERSION_VARIABLE; diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java index 102b0d38461c3..aefb45f6186c1 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java @@ -60,7 +60,6 @@ public Collection createComponents(PluginServices services) { if (enabled) { APMIndexTemplateRegistry registryInstance = registry.get(); registryInstance.setEnabled(APM_DATA_REGISTRY_ENABLED.get(settings)); - clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_DATA_REGISTRY_ENABLED, registryInstance::setEnabled); registryInstance.initialize(); } return Collections.emptyList(); diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java index 34e6dac5a5eca..16de583b0aa2c 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java @@ -16,6 +16,8 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; +import static org.elasticsearch.xpack.oteldata.OTelPlugin.OTEL_DATA_REGISTRY_ENABLED; + public class OTelIndexTemplateRegistry extends YamlTemplateRegistry { public static final String OTEL_TEMPLATE_VERSION_VARIABLE = "xpack.oteldata.template.version"; @@ -31,6 +33,14 @@ public OTelIndexTemplateRegistry( super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); } + @Override + public void initialize() { + super.initialize(); + if(isEnabled()){ + clusterService.getClusterSettings().addSettingsUpdateConsumer(OTEL_DATA_REGISTRY_ENABLED, this::setEnabled); + } + } + @Override protected String getOrigin() { return ClientHelper.OTEL_ORIGIN; diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index f2ff378ab8483..cece2b5373631 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -62,7 +62,6 @@ public Collection createComponents(PluginServices services) { if (enabled) { OTelIndexTemplateRegistry registryInstance = registry.get(); registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings)); - clusterService.getClusterSettings().addSettingsUpdateConsumer(OTEL_DATA_REGISTRY_ENABLED, registryInstance::setEnabled); registryInstance.initialize(); } return Collections.emptyList(); From 3152f992ed511441d7a152b7f22aff4075c29e9d Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Tue, 13 Aug 2024 18:11:10 +0200 Subject: [PATCH 48/51] Fix code-style --- .../elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java | 2 +- .../elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 40edd53a789ec..6f5d4e13dc56b 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -44,7 +44,7 @@ public String getName() { @Override public void initialize() { super.initialize(); - if(isEnabled()){ + if (isEnabled()) { clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_DATA_REGISTRY_ENABLED, this::setEnabled); } } diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java index 16de583b0aa2c..435530542c857 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java @@ -36,7 +36,7 @@ public OTelIndexTemplateRegistry( @Override public void initialize() { super.initialize(); - if(isEnabled()){ + if (isEnabled()) { clusterService.getClusterSettings().addSettingsUpdateConsumer(OTEL_DATA_REGISTRY_ENABLED, this::setEnabled); } } From 680e5197523458cf2640bf563387802d7d06db66 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Wed, 14 Aug 2024 11:50:49 +0200 Subject: [PATCH 49/51] Update x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml Co-authored-by: Felix Barnsteiner --- .../resources/rest-api-spec/test/20_logs.tests.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml index d4dea23022121..d87c2a80deab8 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml @@ -3,17 +3,6 @@ setup: - do: cluster.health: wait_for_events: languid - - do: - cluster.put_component_template: - name: logs-otel@custom - body: - template: - settings: - index: - routing_path: [unit, attributes.*, resource.attributes.*] - mode: time_series - time_series: - start_time: 2024-07-01T13:03:08.138Z --- "Default data_stream.type must be logs": - do: From ef3468f5f6d61b0c4c32f7f1a34698762ad365e1 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Wed, 14 Aug 2024 12:39:19 +0200 Subject: [PATCH 50/51] Enable logsdb --- .../resources/component-templates/logs-otel@mappings.yaml | 5 +++++ .../main/resources/component-templates/otel@mappings.yaml | 2 -- .../semconv-resource-to-ecs@mappings.yaml | 3 +++ .../resources/component-templates/traces-otel@mappings.yaml | 5 +++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index 87f0e517f9fef..a0971f45ccf4f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -4,6 +4,11 @@ _meta: description: Default mappings for OpenTelemetry logs index template installed by x-pack managed: true template: + settings: + index: + mode: logsdb + sort: + field: [ "resource.attributes.host.name" ] mappings: properties: data_stream.type: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml index 3998343ea99f0..fad85661203d6 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml @@ -7,8 +7,6 @@ template: mappings: date_detection: false dynamic: false - _source: - mode: synthetic properties: "@timestamp": #Ultimeately we aim to use date_nanos. Waiting for https://github.com/elastic/elasticsearch/issues/109352 diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml index f72ac9eeb22af..711f72ae95220 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml @@ -14,6 +14,9 @@ template: priority: 30 time_series_dimension: true properties: + host.name: + type: keyword + ignore_above: 1024 telemetry.sdk.language: type: keyword ignore_above: 1024 diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 8a2596e6cf79f..a4c62efeed7a4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -4,6 +4,11 @@ _meta: description: Default mappings for OpenTelemetry traces managed: true template: + settings: + index: + mode: logsdb + sort: + field: [ "resource.attributes.host.name" ] mappings: _source: mode: synthetic From 3bb6737b99f9f3fa89312885607af66aef603904 Mon Sep 17 00:00:00 2001 From: Greg Kalapos Date: Wed, 14 Aug 2024 16:17:16 +0200 Subject: [PATCH 51/51] Update traces@settings.json No lifecycle needed for OTel at this point --- .../template-resources/src/main/resources/traces@settings.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json index b9774bc04d2aa..3f4fdba6f4f46 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json @@ -2,9 +2,6 @@ "template": { "settings": { "index": { - "lifecycle": { - "name": "traces" - }, "codec": "best_compression", "mapping": { "ignore_malformed": true