diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 51bb04185cfde..32b3e44866a6d 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -52,6 +52,10 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("data_stream/30_auto_create_data_stream/Don't initialize failure store during data stream auto-creation on successful index", "Configuring the failure store via data stream templates is not supported anymore.") task.skipTest("data_stream/150_tsdb/TSDB failures go to failure store", "Configuring the failure store via data stream templates is not supported anymore.") + // TODO remove these after removing exact _tsid assertions in 8.x + task.skipTest("data_stream/150_tsdb/dynamic templates", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") + task.skipTest("data_stream/150_tsdb/dynamic templates - conflicting aliases", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") + task.skipTest("data_stream/150_tsdb/dynamic templates with nesting", "The _tsid has changed in a new index version. This tests verifies the exact _tsid value with is too brittle for compatibility testing.") task.skipTest("data_stream/170_modify_data_stream/Modify a data stream's failure store", "Configuring the failure store via data stream templates is not supported anymore.") diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index eaef99d86a86e..eebbeabc75ac1 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; @@ -24,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -34,6 +37,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; @@ -58,13 +62,17 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; +import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; @@ -321,7 +329,10 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { .indexPatterns(List.of("k8s*")) .template( new Template( - Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(), + Settings.builder() + .put("index.mode", "time_series") + .put("index.routing_path", randomBoolean() ? "metricset" : null) + .build(), new CompressedXContent(mappingTemplate), null ) @@ -329,7 +340,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request)); } public void testInvalidTsdbTemplatesMissingSettings() throws Exception { @@ -619,6 +630,207 @@ public void testReindexing() throws Exception { getIndexResponse.getSetting(index2, IndexMetadata.INDEX_ROUTING_PATH.getKey()), equalTo(getIndexResponse.getSetting(index1, IndexMetadata.INDEX_ROUTING_PATH.getKey())) ); + assertThat( + getIndexResponse.getSetting(index2, IndexMetadata.INDEX_DIMENSIONS.getKey()), + equalTo(getIndexResponse.getSetting(index1, IndexMetadata.INDEX_DIMENSIONS.getKey())) + ); + } + + public void testAddDimensionToMapping() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template( + new Template( + Settings.builder().put("index.mode", "time_series").build(), + new CompressedXContent(MAPPING_TEMPLATE), + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + // create data stream + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); + } else { + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + } + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // put mapping with k8s.pod.uid as another time series dimension + var putMappingRequest = new PutMappingRequest(dataStreamName).source(""" + { + "properties": { + "k8s.pod.name": { + "type": "keyword", + "time_series_dimension": true + } + } + } + """, XContentType.JSON); + assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); + } else { + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + } + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat"); + } + + public void testDynamicStringDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + { + "_doc": { + "dynamic_templates": [ + { + "labels": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions custom metadata because the "labels" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + BulkResponse bulkResponse = client().prepareBulk() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(Instant.now())), XContentType.JSON) + ) + .get(); + assertThat(bulkResponse.hasFailures(), is(false)); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + public void testDynamicDimensions() throws Exception { + String dataStreamName = "my-ds"; + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .template(new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent(""" + + { + "_doc": { + "dynamic_templates": [ + { + "label": { + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest)); + + CreateDataStreamAction.Request createDsRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "my-ds" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); + + // doesn't populate index.dimensions because the "label" dynamic template doesn't have a path_math + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // index doc + indexWithPodNames(dataStreamName, Instant.now(), Map.of("k8s.pod.name", "label"), "dog", "cat"); + + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + } + + private void indexWithPodNames(String dataStreamName, Instant timestamp, Map dynamicTemplates, String... podNames) { + // index doc + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (String podName : podNames) { + bulkRequestBuilder.add( + client().prepareIndex(dataStreamName) + .setOpType(DocWriteRequest.OpType.CREATE) + .setSource(DOC.replace("$time", formatInstant(timestamp)).replace("dog", podName), XContentType.JSON) + .request() + .setDynamicTemplates(dynamicTemplates) + ); + } + + BulkResponse bulkResponse = bulkRequestBuilder.get(); + assertThat(bulkResponse.hasFailures(), is(false)); + } + + private T getSetting(String dataStreamName, Setting setting) { + GetIndexResponse getIndexResponse = safeGet( + indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName)) + ); + assertThat(getIndexResponse.getIndices().length, equalTo(1)); + Settings settings = getIndexResponse.getSettings().get(getIndexResponse.getIndices()[0]); + return setting.get(settings); } static String formatInstant(Instant instant) { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index a76dac5db4540..dcee11371cafc 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -44,12 +44,14 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { @@ -185,6 +187,11 @@ public void testIndexingGettingAndSearching() throws Exception { // validate index: var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet(); assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]")); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]")); + } else { + assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), nullValue()); + } // validate mapping var mapping = getIndexResponse.mappings().get(index).getSourceAsMap(); assertMap( diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java index c2c1148124ace..5abaef666a794 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.ClassRule; @@ -28,6 +29,7 @@ public abstract class DisabledSecurityDataStreamTestCase extends ESRestTestCase .distribution(DistributionType.DEFAULT) .setting("xpack.security.enabled", "false") .setting("xpack.watcher.enabled", "false") + .feature(FeatureFlag.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) .build(); @Override diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 9be0c18d18213..d847666ba28b8 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -436,6 +436,10 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception { ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"), containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*") ); + assertThat( + ObjectPath.evaluate(responseBody, "template.settings.index.dimensions"), + containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*") + ); assertThat(ObjectPath.evaluate(responseBody, "overlapping"), empty()); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index dd42101538a0e..20c36bb4188c0 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -13,8 +13,10 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -23,23 +25,23 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.PassThroughObjectMapper; import java.io.IOException; import java.io.UncheckedIOException; import java.time.Instant; import java.util.ArrayList; -import java.util.Iterator; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.function.BiConsumer; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; /** @@ -49,6 +51,8 @@ */ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { + public static final boolean INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG = new FeatureFlag("index_dimensions_tsid_optimization") + .isEnabled(); static final DateFormatter FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; private final CheckedFunction mapperServiceFactory; @@ -122,13 +126,22 @@ public void provideAdditionalMetadata( if (indexTemplateAndCreateRequestSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false && combinedTemplateMappings.isEmpty() == false) { - List routingPaths = findRoutingPaths( + List dimensions = new ArrayList<>(); + boolean matchesAllDimensions = findDimensionFields( indexName, indexTemplateAndCreateRequestSettings, - combinedTemplateMappings + combinedTemplateMappings, + dimensions ); - if (routingPaths.isEmpty() == false) { - additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), routingPaths); + if (dimensions.isEmpty() == false) { + if (matchesAllDimensions && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + // Only set index.dimensions if the paths in the dimensions list match all potential dimension fields. + // This is not the case e.g. if a dynamic template matches by match_mapping_type instead of path_match + additionalSettings.putList(INDEX_DIMENSIONS.getKey(), dimensions); + } + // always populate index.routing_path, so that routing works for older index versions + // this applies to indices created during a rolling upgrade + additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), dimensions); } } } @@ -137,15 +150,60 @@ public void provideAdditionalMetadata( } /** - * Find fields in mapping that are of type keyword and time_series_dimension enabled. + * This is called when mappings are updated, so that the {@link IndexMetadata#getTimeSeriesDimensions()} + * and {@link IndexMetadata#INDEX_ROUTING_PATH} settings are updated to match the new mappings. + * Updates {@link IndexMetadata#getTimeSeriesDimensions} if a new dimension field is added to the mappings, + * or sets {@link IndexMetadata#INDEX_ROUTING_PATH} if a new dimension field is added that doesn't allow for matching all + * dimension fields via a wildcard pattern. + */ + @Override + public void onUpdateMappings( + IndexMetadata indexMetadata, + DocumentMapper documentMapper, + Settings.Builder additionalSettings, + BiConsumer> additionalCustomMetadata + ) { + List indexDimensions = indexMetadata.getTimeSeriesDimensions(); + if (indexDimensions.isEmpty()) { + return; + } + assert indexMetadata.getIndexMode() == IndexMode.TIME_SERIES; + List newIndexDimensions = new ArrayList<>(indexDimensions.size()); + boolean matchesAllDimensions = findDimensionFields(newIndexDimensions, documentMapper); + boolean hasChanges = indexDimensions.size() != newIndexDimensions.size() + && new HashSet<>(indexDimensions).equals(new HashSet<>(newIndexDimensions)) == false; + if (matchesAllDimensions == false) { + // If the new dimensions don't match all potential dimension fields, we need to unset index.dimensions + // so that index.routing_path is used instead. + // This can happen if a new dynamic template is added to an existing index that matches by mapping type instead of path_match. + additionalSettings.putList(INDEX_DIMENSIONS.getKey(), List.of()); + } else if (hasChanges) { + additionalSettings.putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions); + } + } + + /** + * Find fields in mapping that are time_series_dimension enabled. * Using MapperService here has an overhead, but allows the mappings from template to * be merged correctly and fetching the fields without manually parsing the mappings. - * + *

* Alternatively this method can instead parse mappings into map of maps and merge that and * iterate over all values to find the field that can serve as routing value. But this requires * mapping specific logic to exist here. + * + * @param indexName the name of the index for which the dimension fields are being found + * @param allSettings the settings of the index + * @param combinedTemplateMappings the combined mappings from index templates + * (if any) that are applied to the index + * @param dimensions a list to which the found dimension fields will be added + * @return true if all potential dimension fields can be matched via the dimensions in the list, false otherwise */ - private List findRoutingPaths(String indexName, Settings allSettings, List combinedTemplateMappings) { + private boolean findDimensionFields( + String indexName, + Settings allSettings, + List combinedTemplateMappings, + List dimensions + ) { var tmpIndexMetadata = IndexMetadata.builder(indexName); int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(allSettings); @@ -169,57 +227,61 @@ private List findRoutingPaths(String indexName, Settings allSettings, Li // Create MapperService just to extract keyword dimension fields: try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata.build())) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - List routingPaths = new ArrayList<>(); - for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) { - extractPath(routingPaths, fieldMapper); - } - for (var objectMapper : mapperService.documentMapper().mappers().objectMappers().values()) { - if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { - if (passThroughObjectMapper.containsDimensions()) { - routingPaths.add(passThroughObjectMapper.fullPath() + ".*"); - } - } - } - for (var template : mapperService.getAllDynamicTemplates()) { - if (template.pathMatch().isEmpty()) { - continue; - } - - var templateName = "__dynamic__" + template.name(); - var mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - String mappingSnippetType = (String) mappingSnippet.get("type"); - if (mappingSnippetType == null) { - continue; - } + DocumentMapper documentMapper = mapperService.documentMapper(); + return findDimensionFields(dimensions, documentMapper); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - MappingParserContext parserContext = mapperService.parserContext(); - for (Iterator iterator = template.pathMatch().iterator(); iterator.hasNext();) { - var mapper = parserContext.typeParser(mappingSnippetType) - .parse(iterator.next(), mappingSnippet, parserContext) - .build(MapperBuilderContext.root(false, false)); - extractPath(routingPaths, mapper); - if (iterator.hasNext()) { - // Since FieldMapper.parse modifies the Map passed in (removing entries for "type"), that means - // that only the first pathMatch passed in gets recognized as a time_series_dimension. - // To avoid this, each parsing call uses a new mapping snippet. - // Note that a shallow copy of the mappingSnippet map is not enough if there are multi-fields. - mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE); - } + /** + * Finds the dimension fields in the provided document mapper and adds them to the provided list. + * + * @param dimensions the list to which the found dimension fields will be added + * @param documentMapper the document mapper from which to extract the dimension fields + * @return true if all potential dimension fields can be matched via the dimensions in the list, false otherwise + */ + private static boolean findDimensionFields(List dimensions, DocumentMapper documentMapper) { + for (var objectMapper : documentMapper.mappers().objectMappers().values()) { + if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) { + if (passThroughObjectMapper.containsDimensions()) { + dimensions.add(passThroughObjectMapper.fullPath() + ".*"); } } - return routingPaths; - } catch (IOException e) { - throw new UncheckedIOException(e); } + boolean matchesAllDimensions = true; + for (var template : documentMapper.mapping().getRoot().dynamicTemplates()) { + if (template.isTimeSeriesDimension() == false) { + continue; + } + if (template.isSimplePathMatch() == false) { + // If the template is not using a simple path match, the dimensions list can't match all potential dimensions. + // For example, if the dynamic template matches by mapping type (all strings are mapped as dimensions), + // the coordinating node can't rely on the dimensions list to match all dimensions. + // In this case, the index.routing_path setting will be used instead. + matchesAllDimensions = false; + } + if (template.pathMatch().isEmpty() == false) { + dimensions.addAll(template.pathMatch()); + } + } + + for (var fieldMapper : documentMapper.mappers().fieldMappers()) { + extractPath(dimensions, fieldMapper); + } + return matchesAllDimensions; } /** - * Helper method that adds the name of the mapper to the provided list if it is a keyword dimension field. + * Helper method that adds the name of the mapper to the provided list. */ - private static void extractPath(List routingPaths, Mapper mapper) { - if (mapper instanceof KeywordFieldMapper keywordFieldMapper) { - if (keywordFieldMapper.fieldType().isDimension()) { - routingPaths.add(mapper.fullPath()); + private static void extractPath(List dimensions, Mapper mapper) { + if (mapper instanceof FieldMapper fieldMapper && fieldMapper.fieldType().isDimension()) { + String path = mapper.fullPath(); + // don't add if the path already matches via a wildcard pattern in the list + // e.g. if "path.*" is already added, "path.foo" should not be added + if (Regex.simpleMatch(dimensions, path) == false) { + dimensions.add(path); } } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 409ccb2ef2531..ac398c67130f6 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; @@ -19,7 +20,10 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -33,8 +37,9 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER; -import static org.hamcrest.Matchers.contains; +import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; public class DataStreamIndexSettingsProviderTests extends ESTestCase { @@ -70,6 +75,18 @@ public void testGetAdditionalIndexSettings() throws Exception { "field3": { "type": "keyword", "time_series_dimension": true + }, + "field4": { + "type": "long", + "time_series_dimension": true + }, + "field5": { + "type": "ip", + "time_series_dimension": true + }, + "field6": { + "type": "boolean", + "time_series_dimension": true } } } @@ -91,11 +108,16 @@ public void testGetAdditionalIndexSettings() throws Exception { // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throws Exception { @@ -214,11 +236,16 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3")); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGetAdditionalIndexSettingsNoMappings() { @@ -498,11 +525,16 @@ public void testGenerateRoutingPathFromDynamicTemplate() throws Exception { } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntries() throws Exception { @@ -538,7 +570,7 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); @@ -546,8 +578,14 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); - assertEquals(3, routingPathList.size()); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat( + IndexMetadata.INDEX_DIMENSIONS.get(result), + containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") + ); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntriesMultiFields() throws Exception { @@ -588,7 +626,7 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); @@ -596,8 +634,14 @@ public void testGenerateRoutingPathFromDynamicTemplateWithMultiplePathMatchEntri IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") ); - List routingPathList = IndexMetadata.INDEX_ROUTING_PATH.get(result); - assertEquals(3, routingPathList.size()); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat( + IndexMetadata.INDEX_DIMENSIONS.get(result), + containsInAnyOrder("host.id", "xprometheus.labels.*", "yprometheus.labels.*") + ); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() throws Exception { @@ -642,11 +686,16 @@ public void testGenerateRoutingPathFromDynamicTemplate_templateWithNoPathMatch() } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() throws Exception { @@ -693,8 +742,11 @@ public void testGenerateRoutingPathFromDynamicTemplate_nonKeywordTemplate() thro Settings result = generateTsdbSettings(mapping, now); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); - assertEquals(2, IndexMetadata.INDEX_ROUTING_PATH.get(result).size()); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id", "prometheus.labels.*")); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } } public void testGenerateRoutingPathFromPassThroughObject() throws Exception { @@ -706,7 +758,12 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { "labels": { "type": "passthrough", "time_series_dimension": true, - "priority": 2 + "priority": 2, + "properties": { + "label1": { + "type": "keyword" + } + } }, "metrics": { "type": "passthrough", @@ -720,11 +777,121 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*")); + } else { + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + } + } + + public void testAddNewDimension() throws Exception { + String newMapping = """ + { + "_doc": { + "properties": { + "field1": { + "type": "keyword", + "time_series_dimension": true + }, + "field2": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + Settings result = onUpdateMappings("field1", "field1", newMapping); + assertThat(result.size(), equalTo(1)); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field2")); + } + + public void testAddNewDimensionIndexDimensionsUnset() throws Exception { + String newMapping = """ + { + "_doc": { + "properties": { + "field1": { + "type": "keyword", + "time_series_dimension": true + }, + "field2": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + Settings result = onUpdateMappings("field1", null, newMapping); + assertThat(result.size(), equalTo(0)); + } + + public void testAddPassthroughChildField() throws Exception { + String mapping = """ + { + "_doc": { + "properties": { + "labels": { + "type": "passthrough", + "time_series_dimension": true, + "priority": 2, + "properties": { + "label1": { + "type": "keyword" + } + } + }, + "metrics": { + "type": "passthrough", + "priority": 1 + }, + "another_field": { + "type": "keyword" + } + } + } + } + """; + // the new labels.label1 field already matches labels.*, so no change + Settings result = onUpdateMappings("labels.*", "labels.*", mapping); + assertThat(result.size(), equalTo(0)); + } + + public void testAddDynamicTemplate() throws Exception { + String mapping = """ + { + "_doc": { + "dynamic_templates": [ + { + "string_as_dimensions": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ], + "properties": { + "labels": { + "type": "passthrough", + "time_series_dimension": true, + "priority": 1 + } + } + } + } + """; + // the new labels.label1 field already matches labels.*, so no change + Settings result = onUpdateMappings("labels.*", "labels.*", mapping); + assertThat(result.size(), equalTo(1)); + assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); } private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { @@ -750,4 +917,39 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce return builder().put(result).put("index.mode", "time_series").build(); } + private Settings onUpdateMappings(String routingPath, String dimensions, String newMapping) throws IOException { + String dataStreamName = "logs-app1"; + Settings.Builder currentSettings = Settings.builder() + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) + .put(IndexMetadata.INDEX_DIMENSIONS.getKey(), dimensions) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + + IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1)) + .settings(currentSettings) + .build(); + + DocumentMapper documentMapper; + MapperService mapperService = MapperTestUtils.newMapperService( + xContentRegistry(), + createTempDir(), + im.getSettings(), + im.getIndex().getName() + ); + try (mapperService) { + mapperService.merge( + MapperService.SINGLE_MAPPING_NAME, + List.of(new CompressedXContent(newMapping)), + MapperService.MergeReason.INDEX_TEMPLATE + ); + documentMapper = mapperService.documentMapper(); + } + Settings.Builder additionalSettings = builder(); + provider.onUpdateMappings(im, documentMapper, additionalSettings, (k, v) -> {}); + return additionalSettings.build(); + } + } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 598bc90217574..f393fafb5067e 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -148,13 +148,12 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": - requires: cluster_features: ["gte_v8.13.0"] - reason: _tsid hahing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -169,9 +168,8 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - length: {aggregations.tsids.buckets: 2} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -368,7 +366,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -387,7 +384,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -406,7 +402,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -425,7 +420,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: search: @@ -443,7 +437,6 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -520,7 +513,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -539,7 +531,6 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -699,7 +690,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -718,7 +708,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -737,7 +726,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -756,7 +744,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -775,7 +762,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -794,7 +780,6 @@ dynamic templates with nesting: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 193ccd7c968b6..08cf31eb56862 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -419,7 +419,8 @@ static boolean executeBulkItemRequest( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - meteringParserDecorator + meteringParserDecorator, + request.tsid() ); result = primary.applyIndexOperationOnPrimary( version, @@ -737,7 +738,11 @@ private static Engine.Result performOpOnReplica( indexRequest.id(), indexRequest.source(), indexRequest.getContentType(), - indexRequest.routing() + indexRequest.routing(), + Map.of(), + true, + XContentMeteringParserDecorator.NOOP, + indexRequest.tsid() ); result = replica.applyIndexOperationOnReplica( primaryResponse.getSeqNo(), diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 6712920b3bf85..629ec64b414e5 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -207,7 +207,8 @@ private ValidationResult validateMappings( request.routing(), request.getDynamicTemplates(), request.getIncludeSourceOnError(), - XContentMeteringParserDecorator.NOOP + XContentMeteringParserDecorator.NOOP, + request.tsid() ); ProjectMetadata project = projectResolver.getProjectMetadata(clusterService.state()); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 9eef53be5c199..0e6917d1f5ac4 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; @@ -22,6 +23,7 @@ import org.elasticsearch.client.internal.Requests; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.UUIDs; @@ -74,6 +76,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IndexRequest.class); private static final TransportVersion PIPELINES_HAVE_RUN_FIELD_ADDED = TransportVersions.V_8_10_X; + private static final TransportVersion INDEX_REQUEST_INCLUDE_TSID = TransportVersion.fromName("index_request_include_tsid"); private static final Supplier ID_GENERATOR = UUIDs::base64UUID; @@ -147,6 +150,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement * rawTimestamp field is used on the coordinate node, it doesn't need to be serialised. */ private Object rawTimestamp; + private BytesRef tsid; public IndexRequest(StreamInput in) throws IOException { this(null, in); @@ -224,6 +228,10 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { includeSourceOnError = in.readBoolean(); } // else default value is true + + if (in.getTransportVersion().supports(INDEX_REQUEST_INCLUDE_TSID)) { + tsid = in.readBytesRefOrNullIfEmpty(); + } } public IndexRequest() { @@ -363,6 +371,22 @@ public String routing() { return this.routing; } + /** + * When {@link IndexMetadata#INDEX_DIMENSIONS} is populated, + * the coordinating node will calculate _tsid during routing and set it on the request. + * For time series indices where the setting is not populated, the _tsid will be created in the data node during document parsing. + *

+ * The _tsid can not be directly set by a user, it is set by the coordinating node. + */ + public IndexRequest tsid(BytesRef tsid) { + this.tsid = tsid; + return this; + } + + public BytesRef tsid() { + return this.tsid; + } + /** * Sets the ingest pipeline to be executed before indexing the document */ @@ -794,6 +818,9 @@ private void writeBody(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR)) { out.writeBoolean(includeSourceOnError); } + if (out.getTransportVersion().supports(INDEX_REQUEST_INCLUDE_TSID)) { + out.writeBytesRef(tsid); + } } @Override @@ -896,7 +923,7 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, ProjectMetadata project) @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, indexSource.contentType(), indexSource.bytes()); + return indexRouting.indexShard(id, routing, tsid, indexSource.contentType(), indexSource.bytes()); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index f24a2814ea6b7..5c082153e1712 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -514,6 +514,29 @@ public Iterator> settings() { Property.ServerlessPublic ); + /** + * Populated when an index that belongs to a time_series data stream is created or its mappings are updated. + * This setting is used so that the coordinating node knows which fields are time series dimensions + * as it doesn't have access to mappings. + * It's important that this setting is kept up-to-date when new dimensions are added to the mapping. + * The tsid and shard routing for existing time series that don't use the new dimension field won't change as a result of that update. + * When this setting is populated, an optimization kicks in that allows the coordinating node to create the tsid and the routing hash + * in one go. + * Otherwise, the coordinating node only creates the routing hash based on {@link #INDEX_ROUTING_PATH} and the tsid is created + * during document parsing, effectively requiring two passes over the document. + *

+ * The condition for this optimization to kick in is that all possible dimension fields can be identified + * via a list of wildcard patterns. + * If that's not the case (for example when certain types of dynamic templates are used), + * the {@link #INDEX_ROUTING_PATH} is populated instead. + */ + public static final Setting> INDEX_DIMENSIONS = Setting.stringListSetting( + "index.dimensions", + Setting.Property.IndexScope, + Property.Dynamic, + Property.PrivateIndex + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -576,6 +599,7 @@ public Iterator> settings() { private final int routingFactor; private final int routingPartitionSize; private final List routingPaths; + private final List timeSeriesDimensions; private final int numberOfShards; private final int numberOfReplicas; @@ -689,6 +713,7 @@ private IndexMetadata( final int routingNumShards, final int routingPartitionSize, final List routingPaths, + final List timeSeriesDimensions, final ActiveShardCount waitForActiveShards, final ImmutableOpenMap rolloverInfos, final boolean isSystem, @@ -744,6 +769,7 @@ private IndexMetadata( this.routingFactor = routingNumShards / numberOfShards; this.routingPartitionSize = routingPartitionSize; this.routingPaths = routingPaths; + this.timeSeriesDimensions = timeSeriesDimensions; this.waitForActiveShards = waitForActiveShards; this.rolloverInfos = rolloverInfos; this.isSystem = isSystem; @@ -803,6 +829,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.timeSeriesDimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -865,6 +892,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.timeSeriesDimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -935,6 +963,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.timeSeriesDimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -996,6 +1025,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.timeSeriesDimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1052,6 +1082,7 @@ public IndexMetadata withIncrementedVersion() { this.routingNumShards, this.routingPartitionSize, this.routingPaths, + this.timeSeriesDimensions, this.waitForActiveShards, this.rolloverInfos, this.isSystem, @@ -1166,6 +1197,10 @@ public List getRoutingPaths() { return routingPaths; } + public List getTimeSeriesDimensions() { + return timeSeriesDimensions; + } + public int getTotalNumberOfShards() { return totalNumberOfShards; } @@ -2383,6 +2418,7 @@ IndexMetadata build(boolean repair) { } final List routingPaths = INDEX_ROUTING_PATH.get(settings); + final List dimensions = INDEX_DIMENSIONS.get(settings); final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); @@ -2462,6 +2498,7 @@ IndexMetadata build(boolean repair) { getRoutingNumShards(), routingPartitionSize, routingPaths, + dimensions, waitForActiveShards, rolloverInfos.build(), isSystem, diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 1e34128ac4f8d..304383d13e4aa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -42,7 +42,6 @@ import java.util.Base64; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.OptionalInt; import java.util.Set; import java.util.function.IntConsumer; @@ -63,7 +62,7 @@ public abstract class IndexRouting { * Build the routing from {@link IndexMetadata}. */ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { - if (false == metadata.getRoutingPaths().isEmpty()) { + if (metadata.getRoutingPaths().isEmpty() == false || metadata.getTimeSeriesDimensions().isEmpty() == false) { return new ExtractFromSource(metadata); } if (metadata.isRoutingPartitionedIndex()) { @@ -98,7 +97,13 @@ public void postProcess(IndexRequest indexRequest) {} * Called when indexing a document to generate the shard id that should contain * a document with the provided parameters. */ - public abstract int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source); + public abstract int indexShard( + String id, + @Nullable String routing, + @Nullable BytesRef tsid, + XContentType sourceType, + BytesReference source + ); /** * Called when updating a document to generate the shard id that should contain @@ -218,7 +223,13 @@ private static boolean isNewIndexVersion(final IndexVersion creationVersion) { } @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + @Nullable BytesRef tsid, + XContentType sourceType, + BytesReference source + ) { if (id == null) { throw new IllegalStateException("id is required and should have been set by process"); } @@ -307,8 +318,11 @@ public static class ExtractFromSource extends IndexRouting { private final XContentParserConfiguration parserConfig; private final IndexMode indexMode; private final boolean trackTimeSeriesRoutingHash; + private final boolean createTsidDuringRouting; private final boolean addIdWithRoutingHash; private int hash = Integer.MAX_VALUE; + @Nullable + private BytesRef tsid; ExtractFromSource(IndexMetadata metadata) { super(metadata); @@ -316,12 +330,29 @@ public static class ExtractFromSource extends IndexRouting { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } indexMode = metadata.getIndexMode(); - trackTimeSeriesRoutingHash = indexMode == IndexMode.TIME_SERIES - && metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); + assert indexMode != null : "Index mode must be set for ExtractFromSource routing"; + var createTsidDuringRouting = false; + var trackTimeSeriesRoutingHash = false; + List includePaths; + includePaths = metadata.getRoutingPaths(); + if (indexMode == IndexMode.TIME_SERIES) { + if (metadata.getTimeSeriesDimensions().isEmpty() == false + && metadata.getCreationVersion().onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { + // This optimization is only available for new indices where + // the dimensions index setting is automatically populated from the mappings. + // If users manually set the routing paths, the optimization is not applied. + createTsidDuringRouting = true; + includePaths = metadata.getTimeSeriesDimensions(); + } + if (metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { + trackTimeSeriesRoutingHash = true; + } + } + this.createTsidDuringRouting = createTsidDuringRouting; + this.trackTimeSeriesRoutingHash = trackTimeSeriesRoutingHash; addIdWithRoutingHash = indexMode == IndexMode.LOGSDB; - List routingPaths = metadata.getRoutingPaths(); - isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new)); - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(routingPaths), null, true); + isRoutingPath = Regex.simpleMatcher(includePaths.toArray(String[]::new)); + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(includePaths), null, true); } public boolean matchesField(String fieldName) { @@ -330,8 +361,12 @@ public boolean matchesField(String fieldName) { @Override public void postProcess(IndexRequest indexRequest) { - // Update the request with the routing hash, if needed. + // Update the request with the routing hash and the tsid, if needed. // This needs to happen in post-processing, after the routing hash is calculated. + if (createTsidDuringRouting) { + assert tsid != null; + indexRequest.tsid(tsid); + } if (trackTimeSeriesRoutingHash) { indexRequest.routing(TimeSeriesRoutingHashFieldMapper.encode(hash)); } else if (addIdWithRoutingHash) { @@ -341,44 +376,43 @@ public void postProcess(IndexRequest indexRequest) { } @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + @Nullable BytesRef tsid, + XContentType sourceType, + BytesReference source + ) { assert Transports.assertNotTransportThread("parsing the _source can get slow"); checkNoRouting(routing); - hash = hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); + if (createTsidDuringRouting) { + if (tsid == null) { + this.tsid = buildTsid(sourceType, source); + } else { + this.tsid = tsid; + } + hash = hash(this.tsid); + } else { + hash = hashRoutingFields(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); + } int shardId = hashToShardId(hash); return (rerouteWritesIfResharding(shardId)); } public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { - return hashSource(sourceType, source).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); - } - - public String createId(Map flat, byte[] suffix) { - Builder b = builder(); - for (Map.Entry e : flat.entrySet()) { - if (isRoutingPath.test(e.getKey())) { - if (e.getValue() instanceof List listValue) { - for (Object v : listValue) { - b.addHash(e.getKey(), new BytesRef(v.toString())); - } - } else { - b.addHash(e.getKey(), new BytesRef(e.getValue().toString())); - } - } - } - return b.createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); + return hashRoutingFields(sourceType, source).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); } private static int defaultOnEmpty() { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } - public Builder builder() { - return new Builder(); + public RoutingHashBuilder builder() { + return new RoutingHashBuilder(); } - private Builder hashSource(XContentType sourceType, BytesReference source) { - Builder b = builder(); + private RoutingHashBuilder hashRoutingFields(XContentType sourceType, BytesReference source) { + RoutingHashBuilder b = builder(); try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { parser.nextToken(); // Move to first token if (parser.currentToken() == null) { @@ -393,7 +427,24 @@ private Builder hashSource(XContentType sourceType, BytesReference source) { return b; } - public class Builder { + private BytesRef buildTsid(XContentType sourceType, BytesReference source) { + TsidBuilder b = new TsidBuilder(); + try (XContentParser parser = XContentHelper.createParserNotCompressed(parserConfig, source, sourceType)) { + b.add(parser, XContentParserTsidFunnel.get()); + } catch (IOException | ParsingException e) { + throw new IllegalArgumentException("Error extracting tsid: " + e.getMessage(), e); + } + return b.buildTsid(); + } + + /** + * Visible for testing + */ + boolean isCreateTsidDuringRouting() { + return createTsidDuringRouting; + } + + public class RoutingHashBuilder { private final List hashes = new ArrayList<>(); public void addMatching(String fieldName, BytesRef string) { @@ -402,6 +453,11 @@ public void addMatching(String fieldName, BytesRef string) { } } + /** + * Only expected to be called for old indices created before + * {@link IndexVersions#TIME_SERIES_ROUTING_HASH_IN_ID} while creating (during ingestion) + * or synthesizing (at query time) the _id field. + */ public String createId(byte[] suffix, IntSupplier onEmpty) { byte[] idBytes = new byte[4 + suffix.length]; ByteUtils.writeIntLE(buildHash(onEmpty), idBytes, 0); @@ -472,6 +528,16 @@ private int buildHash(IntSupplier onEmpty) { } return hash; } + + private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { + @Override + public int compareTo(NameAndHash o) { + int i = name.compareTo(o.name); + if (i != 0) return i; + // ensures array values are in the order as they appear in the source + return Integer.compare(order, o.order); + } + } } private static int hash(BytesRef ref) { @@ -533,14 +599,4 @@ private String error(String operation) { return operation + " is not supported because the destination index [" + indexName + "] is in " + indexMode.getName() + " mode"; } } - - private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { - @Override - public int compareTo(NameAndHash o) { - int i = name.compareTo(o.name); - if (i != 0) return i; - // ensures array values are in the order as they appear in the source - return Integer.compare(order, o.order); - } - } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index f5c8a96d35f49..b45f3ccbdc5f3 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -203,6 +203,14 @@ public BytesRef readBytesRef() throws IOException { return readBytesRef(length); } + public @Nullable BytesRef readBytesRefOrNullIfEmpty() throws IOException { + int length = readArraySize(); + if (length == 0) { + return null; + } + return readBytesRef(length); + } + public BytesRef readBytesRef(int length) throws IOException { if (length == 0) { return new BytesRef(); diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 93ddb5d3fc485..a0a28e9322956 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -223,6 +223,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // TSDB index settings IndexSettings.MODE, IndexMetadata.INDEX_ROUTING_PATH, + IndexMetadata.INDEX_DIMENSIONS, IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME, IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 07cbc39ca103b..40231ac9e3d6c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.RoutingFields; import org.elasticsearch.index.mapper.RoutingPathFields; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; @@ -113,7 +114,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -146,13 +147,17 @@ void validateWithOtherSettings(Map, Object> settings) { throw new IllegalArgumentException(error(unsupported)); } } - checkSetting(settings, IndexMetadata.INDEX_ROUTING_PATH); + Setting> routingPath = IndexMetadata.INDEX_ROUTING_PATH; + if (isEmpty(settings, routingPath) && isEmpty(settings, IndexMetadata.INDEX_DIMENSIONS)) { + // index.dimensions is a private setting that only gets populated for data streams. + // We don't include it in the error message to not confuse users that are manually creating time series indices + // which is the only case where this error can occur. + throw new IllegalArgumentException(tsdbMode() + " requires a non-empty [" + routingPath.getKey() + "]"); + } } - private static void checkSetting(Map, Object> settings, Setting setting) { - if (Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) { - throw new IllegalArgumentException(tsdbMode() + " requires a non-empty [" + setting.getKey() + "]"); - } + private static boolean isEmpty(Map, Object> settings, Setting> setting) { + return Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting)); } private static String error(Setting unsupported) { @@ -213,7 +218,11 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { + if (source.tsid() != null) { + // If the source already has a _tsid field, we don't need to extract routing from the source. + return RoutingFields.Noop.INSTANCE; + } IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); return new RoutingPathFields(routing.builder()); } @@ -294,7 +303,7 @@ public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -375,7 +384,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public RoutingFields buildRoutingFields(IndexSettings settings) { + public RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source) { return RoutingFields.Noop.INSTANCE; } @@ -454,6 +463,7 @@ private static CompressedXContent createDefaultMapping(boolean includeHostName) IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, IndexMetadata.INDEX_ROUTING_PATH, + IndexMetadata.INDEX_DIMENSIONS, IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME @@ -530,7 +540,7 @@ public String getName() { /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ - public abstract RoutingFields buildRoutingFields(IndexSettings settings); + public abstract RoutingFields buildRoutingFields(IndexSettings settings, SourceToParse source); /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 1c710e2e36c14..f3985c2dd5f8f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -183,6 +183,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2); public static final IndexVersion EXCLUDE_SOURCE_VECTORS_DEFAULT = def(9_035_0_00, Version.LUCENE_10_2_2); public static final IndexVersion DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB = def(9_036_0_00, Version.LUCENE_10_2_2); + public static final IndexVersion TSID_CREATED_DURING_ROUTING = def(9_037_0_00, Version.LUCENE_10_2_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index c1c1f8ed0f517..edf60c69460f6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -12,11 +12,13 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -94,7 +96,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL ) ) ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser, source.tsid()); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -1069,12 +1071,14 @@ private static class RootDocumentParserContext extends DocumentParserContext { private final long maxAllowedNumNestedDocs; private long numNestedDocs; private boolean docsReversed = false; + private final BytesRef tsid; RootDocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - XContentParser parser + XContentParser parser, + BytesRef tsid ) throws IOException { super( mappingLookup, @@ -1083,6 +1087,9 @@ private static class RootDocumentParserContext extends DocumentParserContext { mappingLookup.getMapping().getRoot(), ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); + this.tsid = tsid; + assert tsid == null || mappingParserContext.getIndexSettings().getMode() == IndexMode.TIME_SERIES + : "tsid should only be set for time series indices"; if (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) { this.parser = DotExpandingXContentParser.expandDots(parser, this.path); } else { @@ -1140,6 +1147,11 @@ protected void addDoc(LuceneDocument doc) { this.documents.add(doc); } + @Override + public BytesRef getTsid() { + return this.tsid; + } + @Override public Iterable nonRootDocuments() { if (docsReversed) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 69faec23d2d48..28cd1c4533db0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -116,6 +117,11 @@ public XContentParser.Token getImmediateXContentParent() { public boolean isImmediateParentAnArray() { return in.isImmediateParentAnArray(); } + + @Override + public BytesRef getTsid() { + return in.getTsid(); + } } /** @@ -265,7 +271,7 @@ protected DocumentParserContext( null, null, SeqNoFieldMapper.SequenceIDFields.emptySeqID(mappingParserContext.getIndexSettings().seqNoIndexOptions()), - RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings()), + RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings(), source), parent, dynamic, new HashSet<>(), @@ -865,6 +871,9 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() { protected abstract void addDoc(LuceneDocument doc); + @Nullable + public abstract BytesRef getTsid(); + /** * Find a dynamic mapping template for the given field and its matching type * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index 7c58e58390503..f661927e7cbd6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -31,6 +31,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM; + public class DynamicTemplate implements ToXContentObject { public enum MatchType { @@ -447,6 +449,24 @@ public List match() { return match; } + public boolean isTimeSeriesDimension() { + if (mapping != null) { + Object value = mapping.get(TIME_SERIES_DIMENSION_PARAM); + if (value instanceof Boolean bool) return bool; + } + return false; + } + + public boolean isSimplePathMatch() { + return pathMatch.isEmpty() == false + && pathUnmatch.isEmpty() + && match.isEmpty() + && unmatch.isEmpty() + && matchMappingType.isEmpty() + && unmatchMappingType.isEmpty() + && matchType != MatchType.REGEX; + } + public boolean match(String templateName, String path, String fieldName, XContentFieldType xcontentFieldType) { // If the template name parameter is specified, then we will check only the name of the template and ignore other matches. if (templateName != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 741252f98473b..12228760faf35 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -67,9 +67,10 @@ final class TsIdLoader implements IdLoader { } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { - IndexRouting.ExtractFromSource.Builder[] builders = null; + IndexRouting.ExtractFromSource.RoutingHashBuilder[] builders = null; if (indexRouting != null) { - builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; + // this branch is for legacy indices before IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID + builders = new IndexRouting.ExtractFromSource.RoutingHashBuilder[docIdsInLeaf.length]; for (int i = 0; i < builders.length; i++) { builders[i] = indexRouting.builder(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index ebf4fb0d11cc6..3781242815e95 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -680,7 +680,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } private void indexValue(DocumentParserContext context, ESInetAddressPoint address) { - if (dimension) { + if (dimension && context.getRoutingFields().isNoop() == false) { context.getRoutingFields().addIp(fieldType().name(), address.getInetAddress()); } LuceneDocument doc = context.doc(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java index 4d8d8fdcbd296..0ad4f4e0b49aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFields.java @@ -22,8 +22,8 @@ public interface RoutingFields { /** * Collect routing fields from index settings */ - static RoutingFields fromIndexSettings(IndexSettings indexSettings) { - return indexSettings.getMode().buildRoutingFields(indexSettings); + static RoutingFields fromIndexSettings(IndexSettings indexSettings, SourceToParse source) { + return indexSettings.getMode().buildRoutingFields(indexSettings, source); } /** @@ -45,6 +45,8 @@ default RoutingFields addString(String fieldName, String value) { RoutingFields addBoolean(String fieldName, boolean value); + boolean isNoop(); + /** * Noop implementation that doesn't perform validations on routing fields */ @@ -81,5 +83,10 @@ public RoutingFields addUnsignedLong(String fieldName, long value) { public RoutingFields addBoolean(String fieldName, boolean value) { return this; } + + @Override + public boolean isNoop() { + return true; + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java index 73baca1bf3fdb..361163ea1be66 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingPathFields.java @@ -59,9 +59,9 @@ public final class RoutingPathFields implements RoutingFields { * Builds the routing. Used for building {@code _id}. If null then skipped. */ @Nullable - private final IndexRouting.ExtractFromSource.Builder routingBuilder; + private final IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder; - public RoutingPathFields(@Nullable IndexRouting.ExtractFromSource.Builder routingBuilder) { + public RoutingPathFields(@Nullable IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder) { this.routingBuilder = routingBuilder; } @@ -69,7 +69,7 @@ SortedMap> routingValues() { return Collections.unmodifiableSortedMap(routingValues); } - IndexRouting.ExtractFromSource.Builder routingBuilder() { + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder() { return routingBuilder; } @@ -207,6 +207,11 @@ public RoutingFields addBoolean(String fieldName, boolean value) { return this; } + @Override + public boolean isNoop() { + return false; + } + private void add(String fieldName, BytesReference encoded) throws IOException { BytesRef name = new BytesRef(fieldName); List values = routingValues.get(name); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 5396fdef0f041..8a2ecb126c6cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Nullable; @@ -24,6 +25,8 @@ public class SourceToParse { private final String id; + private final @Nullable BytesRef tsid; + private final @Nullable String routing; private final XContentType xContentType; @@ -41,7 +44,8 @@ public SourceToParse( @Nullable String routing, Map dynamicTemplates, boolean includeSourceOnError, - XContentMeteringParserDecorator meteringParserDecorator + XContentMeteringParserDecorator meteringParserDecorator, + @Nullable BytesRef tsid ) { this.id = id; // we always convert back to byte array, since we store it and Field only supports bytes.. @@ -52,14 +56,15 @@ public SourceToParse( this.dynamicTemplates = Objects.requireNonNull(dynamicTemplates); this.includeSourceOnError = includeSourceOnError; this.meteringParserDecorator = meteringParserDecorator; + this.tsid = tsid; } public SourceToParse(String id, BytesReference source, XContentType xContentType) { - this(id, source, xContentType, null, Map.of(), true, XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, null, Map.of(), true, XContentMeteringParserDecorator.NOOP, null); } public SourceToParse(String id, BytesReference source, XContentType xContentType, String routing) { - this(id, source, xContentType, routing, Map.of(), true, XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, routing, Map.of(), true, XContentMeteringParserDecorator.NOOP, null); } public SourceToParse( @@ -67,9 +72,10 @@ public SourceToParse( BytesReference source, XContentType xContentType, String routing, - Map dynamicTemplates + Map dynamicTemplates, + BytesRef tsid ) { - this(id, source, xContentType, routing, dynamicTemplates, true, XContentMeteringParserDecorator.NOOP); + this(id, source, xContentType, routing, dynamicTemplates, true, XContentMeteringParserDecorator.NOOP, tsid); } public BytesReference source() { @@ -113,4 +119,8 @@ public XContentMeteringParserDecorator getMeteringParserDecorator() { public boolean getIncludeSourceOnError() { return includeSourceOnError; } + + public BytesRef tsid() { + return tsid; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index b94fa64b42428..0f3776f1db9bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -14,6 +14,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -150,16 +151,21 @@ private TimeSeriesIdFieldMapper(boolean useDocValuesSkipper) { public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; - final RoutingPathFields routingPathFields = (RoutingPathFields) context.getRoutingFields(); final BytesRef timeSeriesId; + final RoutingPathFields routingPathFields; if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING)) { + routingPathFields = (RoutingPathFields) context.getRoutingFields(); long limit = context.indexSettings().getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); int size = routingPathFields.routingValues().size(); if (size > limit) { throw new MapperException("Too many dimension fields [" + size + "], max [" + limit + "] dimension fields allowed"); } timeSeriesId = buildLegacyTsid(routingPathFields).toBytesRef(); + } else if (context.getTsid() != null) { + routingPathFields = null; + timeSeriesId = context.getTsid(); } else { + routingPathFields = (RoutingPathFields) context.getRoutingFields(); timeSeriesId = routingPathFields.buildHash().toBytesRef(); } @@ -169,13 +175,15 @@ public void postParse(DocumentParserContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); } - BytesRef uidEncoded = TsidExtractingIdFieldMapper.createField( - context, - getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) - ? routingPathFields.routingBuilder() - : null, - timeSeriesId - ); + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder; + if (getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) && routingPathFields != null) { + // For legacy indices, we need to create the routing hash from the routing path fields. + routingBuilder = routingPathFields.routingBuilder(); + } else { + // For newer indices, the routing hash is stored in SourceToParse#routing, so we can use that directly. + routingBuilder = null; + } + BytesRef uidEncoded = TsidExtractingIdFieldMapper.createField(context, routingBuilder, timeSeriesId); // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 8aca004949209..b48b22520ea36 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -48,7 +48,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext public static BytesRef createField( DocumentParserContext context, - IndexRouting.ExtractFromSource.Builder routingBuilder, + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder, BytesRef tsid ) { final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc()); @@ -115,7 +115,7 @@ public static String createId(int routingHash, BytesRef tsid, long timestamp) { public static String createId( boolean dynamicMappersExists, - IndexRouting.ExtractFromSource.Builder routingBuilder, + IndexRouting.ExtractFromSource.RoutingHashBuilder routingBuilder, BytesRef tsid, long timestamp, byte[] suffix diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 93ef04ddd159a..953ac8f7a6941 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -177,7 +177,7 @@ private void addField(Context context, ContentPath path, String currentName, Str fields.add(new SortedSetDocValuesField(rootFieldFullPath, bytesValue)); fields.add(new SortedSetDocValuesField(keyedFieldFullPath, bytesKeyedValue)); - if (fieldType.isDimension() == false) { + if (fieldType.isDimension() == false || context.documentParserContext().getRoutingFields().isNoop()) { return; } diff --git a/server/src/main/resources/transport/definitions/referable/index_request_include_tsid.csv b/server/src/main/resources/transport/definitions/referable/index_request_include_tsid.csv new file mode 100644 index 0000000000000..d5d0058b471e2 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_request_include_tsid.csv @@ -0,0 +1 @@ +9167000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index a3190bb37c6a2..bf1a90e5be4e9 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -inference_results_map_with_cluster_alias,9166000 +index_request_include_tsid,9167000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index c1abc120d4cb1..d9a2d885b82a6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -553,14 +553,21 @@ private static IndexMetadata createIndexMetadata( boolean frozen, IndexMode mode ) { + IndexMetadata.Builder indexBuilder = IndexMetadata.builder(name); Settings.Builder settingsBuilder = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .put("index.hidden", hidden) .put("index.frozen", frozen) .put("index.mode", mode.toString()); - IndexMetadata.Builder indexBuilder = IndexMetadata.builder(name) - .settings(settingsBuilder) + if (mode == IndexMode.TIME_SERIES) { + settingsBuilder.put( + randomBoolean() ? IndexMetadata.INDEX_DIMENSIONS.getKey() : IndexMetadata.INDEX_ROUTING_PATH.getKey(), + "dummy_value" + ); + } + + indexBuilder.settings(settingsBuilder) .state(closed ? IndexMetadata.State.CLOSE : IndexMetadata.State.OPEN) .system(system) .numberOfShards(1) diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index 9f5c9db1452d9..0373f9975d9df 100644 --- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.action.index; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; @@ -480,6 +481,7 @@ public void testSerialization() throws IOException { assertThat(copy.getFinalPipeline(), equalTo(indexRequest.getFinalPipeline())); assertThat(copy.ifPrimaryTerm(), equalTo(indexRequest.ifPrimaryTerm())); assertThat(copy.isRequireDataStream(), equalTo(indexRequest.isRequireDataStream())); + assertThat(copy.tsid(), equalTo(indexRequest.tsid())); } private IndexRequest createTestInstance() { @@ -495,6 +497,7 @@ private IndexRequest createTestInstance() { for (int i = 0; i < randomIntBetween(0, 20); i++) { indexRequest.addPipeline(randomAlphaOfLength(20)); } + indexRequest.tsid(randomFrom(new BytesRef(randomAlphaOfLength(20)), null)); return indexRequest; } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 63ebe1a855211..4733d47450885 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -1185,6 +1185,7 @@ public void testValidate() { .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start3.toEpochMilli()) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end3.toEpochMilli()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dummy_path") .build() ) .build() diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 570a4ac1a7c3b..d9258be24398a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentType; @@ -47,6 +46,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.stringContainsInOrder; public class IndexRoutingTests extends ESTestCase { public void testSimpleRoutingRejectsEmptyId() { @@ -464,7 +464,7 @@ public void testRequiredRouting() { */ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable String routing) { return switch (between(0, 3)) { - case 0 -> indexRouting.indexShard(id, routing, null, null); + case 0 -> indexRouting.indexShard(id, routing, null, null, null); case 1 -> indexRouting.updateShard(id, routing); case 2 -> indexRouting.deleteShard(id, routing); case 3 -> indexRouting.getShard(id, routing); @@ -497,18 +497,18 @@ public void testRoutingPathEmptySource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of())) + () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source(Map.of())) ); - assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); + assertThat(e.getMessage(), stringContainsInOrder("Error extracting", "source didn't contain any")); } public void testRoutingPathMismatchSource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of("bar", "dog"))) + () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source(Map.of("bar", "dog"))) ); - assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); + assertThat(e.getMessage(), stringContainsInOrder("Error extracting", "source didn't contain any")); } public void testRoutingPathUpdate() throws IOException { @@ -527,7 +527,7 @@ public void testRoutingIndexWithRouting() throws IOException { String docRouting = randomAlphaOfLength(5); Exception e = expectThrows( IllegalArgumentException.class, - () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source) + () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, null, XContentType.JSON, source) ); assertThat( e.getMessage(), @@ -547,7 +547,7 @@ public void testRoutingPathCollectSearchWithRouting() throws IOException { public void testRoutingPathOneTopLevel() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo"); - assertIndexShard(routing, Map.of("foo", "cat", "bar", "dog"), Math.floorMod(hash(List.of("foo", "cat")), shards)); + assertIndexShard(routing, Map.of("foo", "cat", "bar", "dog"), List.of("foo", "cat"), shards); } public void testRoutingPathManyTopLevel() throws IOException { @@ -556,18 +556,15 @@ public void testRoutingPathManyTopLevel() throws IOException { assertIndexShard( routing, Map.of("foo", "cat", "bar", "dog", "foa", "a", "fob", "b"), - Math.floorMod(hash(List.of("foa", "a", "fob", "b", "foo", "cat")), shards) // Note that the fields are sorted + List.of("foa", "a", "fob", "b", "foo", "cat"), + shards // Note that the fields are sorted ); } public void testRoutingPathOneSub() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo.*"); - assertIndexShard( - routing, - Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), - Math.floorMod(hash(List.of("foo.bar", "cat")), shards) - ); + assertIndexShard(routing, Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), List.of("foo.bar", "cat"), shards); } public void testRoutingPathManySubs() throws IOException { @@ -576,31 +573,32 @@ public void testRoutingPathManySubs() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("a", "cat"), "bar", Map.of("thing", "yay", "this", "too")), - Math.floorMod(hash(List.of("bar.thing", "yay", "bar.this", "too", "foo.a", "cat")), shards) + List.of("bar.thing", "yay", "bar.this", "too", "foo.a", "cat"), + shards ); } public void testRoutingPathDotInName() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo.bar"); - assertIndexShard(routing, Map.of("foo.bar", "cat", "baz", "dog"), Math.floorMod(hash(List.of("foo.bar", "cat")), shards)); + assertIndexShard(routing, Map.of("foo.bar", "cat", "baz", "dog"), List.of("foo.bar", "cat"), shards); } public void testRoutingPathNumbersInSource() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo"); long randomLong = randomLong(); - assertIndexShard(routing, Map.of("foo", randomLong), Math.floorMod(hash(List.of("foo", Long.toString(randomLong))), shards)); + assertIndexShard(routing, Map.of("foo", randomLong), List.of("foo", randomLong), shards); double randomDouble = randomDouble(); - assertIndexShard(routing, Map.of("foo", randomDouble), Math.floorMod(hash(List.of("foo", Double.toString(randomDouble))), shards)); - assertIndexShard(routing, Map.of("foo", 123), Math.floorMod(hash(List.of("foo", "123")), shards)); + assertIndexShard(routing, Map.of("foo", randomDouble), List.of("foo", randomDouble), shards); + assertIndexShard(routing, Map.of("foo", 123), List.of("foo", 123), shards); } public void testRoutingPathBooleansInSource() throws IOException { int shards = between(2, 1000); IndexRouting routing = indexRoutingForPath(shards, "foo"); - assertIndexShard(routing, Map.of("foo", true), Math.floorMod(hash(List.of("foo", "true")), shards)); - assertIndexShard(routing, Map.of("foo", false), Math.floorMod(hash(List.of("foo", "false")), shards)); + assertIndexShard(routing, Map.of("foo", true), List.of("foo", true), shards); + assertIndexShard(routing, Map.of("foo", false), List.of("foo", false), shards); } public void testRoutingPathArraysInSource() throws IOException { @@ -610,7 +608,8 @@ public void testRoutingPathArraysInSource() throws IOException { routing, Map.of("c", List.of(true), "d", List.of(), "a", List.of("foo", "bar", "foo"), "b", List.of(21, 42)), // Note that the fields are sorted - Math.floorMod(hash(List.of("a", "foo", "a", "bar", "a", "foo", "b", "21", "b", "42", "c", "true")), shards) + List.of("a", "foo", "a", "bar", "a", "foo", "b", 21, "b", 42, "c", true), + shards ); } @@ -621,19 +620,18 @@ public void testRoutingPathObjectArraysInSource() throws IOException { BytesReference source = source(Map.of("a", List.of("foo", Map.of("foo", "bar")))); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source) + () -> routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, source) ); assertThat( e.getMessage(), - equalTo("Error extracting routing: Failed to parse object: expecting value token but found [START_OBJECT]") + stringContainsInOrder("Error extracting", "Failed to parse object: expecting value token but found [START_OBJECT]") ); } public void testRoutingPathBwc() throws IOException { - IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); - IndexRouting routing = indexRoutingForPath(version, 8, "dim.*,other.*,top"); + IndexRouting routing = indexRoutingForRoutingPath(IndexVersion.current(), 8, "dim.*,other.*,top"); /* - * These when we first added routing_path. If these values change + * These are the expected shards when we first added routing_path. If these values change * time series will be routed to unexpected shards. You may modify * them with a new index created version, but when you do you must * copy this test and patch the versions at the top. Because newer @@ -649,6 +647,29 @@ public void testRoutingPathBwc() throws IOException { assertIndexShard(routing, Map.of("dim.a", "a"), 4); } + public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { + IndexRouting routing = indexRoutingForTimeSeriesDimensions(IndexVersion.current(), 8, "dim.*,other.*,top"); + /* + * These are the expected shards after tsid based routing. If these values change + * time series will be routed to unexpected shards. You may modify + * them with a new index created version, but when you do you must + * copy this test and patch the versions at the top. Because newer + * versions of Elasticsearch must continue to route based on the + * version on the index. + */ + assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 7); + assertIndexShard(routing, Map.of("dim", Map.of("a", "b")), 5); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d")), 5); + assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 0); + assertIndexShard(routing, Map.of("top", "a"), 7); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 2); + assertIndexShard(routing, Map.of("dim.a", "a"), 7); + assertIndexShard(routing, Map.of("dim.a", 1), 0); + assertIndexShard(routing, Map.of("dim.a", "1"), 5); + assertIndexShard(routing, Map.of("dim.a", true), 5); + assertIndexShard(routing, Map.of("dim.a", "true"), 6); + } + public void testRoutingPathReadWithInvalidString() throws IOException { int shards = between(2, 1000); IndexRouting indexRouting = indexRoutingForPath(shards, "foo"); @@ -682,9 +703,9 @@ public void testRoutingPathLogsdb() throws IOException { assertNull(req.id()); // Verify that routing uses the field name and value in the routing path. - int expectedShard = Math.floorMod(hash(List.of("foo", "A")), shards); + int expectedShard = expectedShard(routing, List.of("foo", "A"), shards); BytesReference sourceBytes = source(Map.of("foo", "A", "bar", "B")); - assertEquals(expectedShard, routing.indexShard(null, null, XContentType.JSON, sourceBytes)); + assertEquals(expectedShard, routing.indexShard(null, null, null, XContentType.JSON, sourceBytes)); // Verify that the request id gets updated to contain the routing hash. routing.postProcess(req); @@ -705,7 +726,7 @@ public void testCollectSearchShardsUnpartitionedWithResharding() throws IOExcept var shardToRouting = new HashMap(); do { var routing = randomAlphaOfLength(5); - var shard = initialRouting.indexShard("dummy", routing, null, null); + var shard = initialRouting.indexShard("dummy", routing, null, null, null); if (shardToRouting.containsKey(shard) == false) { shardToRouting.put(shard, routing); } @@ -785,7 +806,7 @@ public void testCollectSearchShardsPartitionedWithResharding() throws IOExceptio var shardToRouting = new TreeMap(); do { var routing = randomAlphaOfLength(5); - var shard = initialRouting.indexShard("dummy", routing, null, null); + var shard = initialRouting.indexShard("dummy", routing, null, null, null); if (shardToRouting.containsKey(shard) == false) { shardToRouting.put(shard, routing); } @@ -889,32 +910,52 @@ private IndexRouting indexRoutingForPath(int shards, String path) { return indexRoutingForPath(IndexVersion.current(), shards, path); } - private IndexRouting indexRoutingForPath(IndexVersion createdVersion, int shards, String path) { + private IndexRouting indexRoutingForPath(IndexVersion indexVersion, int shards, String path) { + // old way of routing paths created during routing + // current way of routing paths created during routing via tsid + String setting = randomBoolean() ? IndexMetadata.INDEX_DIMENSIONS.getKey() : IndexMetadata.INDEX_ROUTING_PATH.getKey(); + return getIndexRoutingWithSetting(indexVersion, shards, path, setting); + } + + private IndexRouting indexRoutingForRoutingPath(IndexVersion createdVersion, int shards, String path) { + return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_ROUTING_PATH.getKey()); + } + + private IndexRouting indexRoutingForTimeSeriesDimensions(IndexVersion createdVersion, int shards, String path) { + return getIndexRoutingWithSetting(createdVersion, shards, path, IndexMetadata.INDEX_DIMENSIONS.getKey()); + } + + private static IndexRouting getIndexRoutingWithSetting(IndexVersion indexVersion, int shards, String path, String setting) { return IndexRouting.fromIndexMetadata( IndexMetadata.builder("test") - .settings( - settings(createdVersion).put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), path) - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .build() - ) + .settings(settings(indexVersion).put(setting, path).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)) .numberOfShards(shards) .numberOfReplicas(1) .build() ); } + private void assertIndexShard(IndexRouting routing, Map source, List keysAndValues, int shards) + throws IOException { + assertIndexShard(routing, source, expectedShard(routing, keysAndValues, shards)); + } + private void assertIndexShard(IndexRouting routing, Map source, int expectedShard) throws IOException { byte[] suffix = randomSuffix(); BytesReference sourceBytes = source(source); - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); + assertThat(routing.indexShard(randomAlphaOfLength(5), null, null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); IndexRouting.ExtractFromSource r = (IndexRouting.ExtractFromSource) routing; + if (r.isCreateTsidDuringRouting()) { + // The rest of the assertions are only relevant when only the routing hash is created + return; + } String idFromSource = r.createId(XContentType.JSON, sourceBytes, suffix); assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); Map flattened = flatten(source); - String idFromFlattened = r.createId(flattened, suffix); + String idFromFlattened = r.createId(XContentType.JSON, sourceBytes, suffix); assertThat(idFromFlattened, equalTo(idFromSource)); - IndexRouting.ExtractFromSource.Builder b = r.builder(); + IndexRouting.ExtractFromSource.RoutingHashBuilder b = r.builder(); for (Map.Entry e : flattened.entrySet()) { if (e.getValue() instanceof List listValue) { listValue.forEach(v -> b.addMatching(e.getKey(), new BytesRef(v.toString()))); @@ -965,17 +1006,50 @@ private void flatten(Map result, String path, Map m) { } } + private int expectedShard(IndexRouting routing, List keysAndValues, int shards) { + return Math.floorMod(hash(routing, keysAndValues), shards); + } + /** * Build the hash we expect from the extracter. */ - private int hash(List keysAndValues) { + private int hash(IndexRouting routing, List keysAndValues) { + if (routing instanceof IndexRouting.ExtractFromSource extractFromSource && extractFromSource.isCreateTsidDuringRouting()) { + return tsidBasedRoutingHash(keysAndValues); + } + return legacyRoutingHash(keysAndValues); + } + + private int legacyRoutingHash(List keysAndValues) { assertThat(keysAndValues.size() % 2, equalTo(0)); int hash = 0; for (int i = 0; i < keysAndValues.size(); i += 2) { - int keyHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i)), 0); - int valueHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i + 1)), 0); + int keyHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i).toString()), 0); + int valueHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i + 1).toString()), 0); hash = hash * 31 + (keyHash ^ valueHash); } return hash; } + + private static int tsidBasedRoutingHash(List keysAndValues) { + TsidBuilder tsidBuilder = new TsidBuilder(); + for (int i = 0; i < keysAndValues.size(); i += 2) { + String key = keysAndValues.get(i).toString(); + Object value = keysAndValues.get(i + 1); + if (value instanceof String sValue) { + tsidBuilder.addStringDimension(key, sValue); + } else if (value instanceof Boolean bValue) { + tsidBuilder.addBooleanDimension(key, bValue); + } else if (value instanceof Integer iValue) { + tsidBuilder.addIntDimension(key, iValue); + } else if (value instanceof Long lValue) { + tsidBuilder.addLongDimension(key, lValue); + } else if (value instanceof Double dValue) { + tsidBuilder.addDoubleDimension(key, dValue); + } else { + throw new IllegalArgumentException("Unsupported value type for TSID routing: " + value.getClass()); + } + } + return StringHelper.murmurhash3_x86_32(tsidBuilder.buildTsid(), 0); + } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 93dedbb355fbd..7cbc391ceda2f 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -27,6 +27,7 @@ import static org.elasticsearch.index.IndexSettings.TIME_SERIES_END_TIME; import static org.elasticsearch.index.IndexSettings.TIME_SERIES_START_TIME; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class TimeSeriesModeTests extends MapperServiceTestCase { @@ -70,16 +71,20 @@ public void testSortOrder() { public void testWithoutRoutingPath() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] requires a non-empty [index.routing_path]")); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) + ); + assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); } public void testWithEmptyRoutingPath() { Settings s = getSettings(""); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] requires a non-empty [index.routing_path]")); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> new IndexSettings(IndexSettingsTests.newIndexMeta("test", s), Settings.EMPTY) + ); + assertThat(e.getMessage(), containsString("[index.mode=time_series] requires a non-empty [index.routing_path]")); } public void testWithoutStartTime() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index a771746fde1db..034924e72c7be 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -666,7 +666,7 @@ public void testTemplateWithoutMatchPredicates() throws Exception { {"foo": "41.12,-71.34", "bar": "41.12,-71.34"} """; ParsedDocument doc = mapperService.documentMapper() - .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point"))); + .parse(new SourceToParse("1", new BytesArray(json), XContentType.JSON, null, Map.of("foo", "geo_point"), null)); assertThat(doc.rootDoc().getFields("foo"), hasSize(1)); assertThat(doc.rootDoc().getFields("bar"), hasSize(1)); } diff --git a/server/src/test/java/org/elasticsearch/indices/TimestampFieldMapperServiceTests.java b/server/src/test/java/org/elasticsearch/indices/TimestampFieldMapperServiceTests.java index 7ef1c56f27a4a..e1b0c24c1720c 100644 --- a/server/src/test/java/org/elasticsearch/indices/TimestampFieldMapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/TimestampFieldMapperServiceTests.java @@ -92,12 +92,19 @@ private static ClusterState initialClusterState() { } private static IndexMetadata createIndex(boolean isTimeSeries) { - return IndexMetadata.builder(randomAlphaOfLength(5)) - .settings( - indexSettings(IndexVersion.current(), 1, 0).put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") - .put(IndexSettings.MODE.getKey(), isTimeSeries ? IndexMode.TIME_SERIES.getName() : IndexMode.STANDARD.getName()) - ) - .build(); + Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), 1, 0).put( + IndexSettings.TIME_SERIES_START_TIME.getKey(), + "2021-04-28T00:00:00Z" + ) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .put(IndexSettings.MODE.getKey(), isTimeSeries ? IndexMode.TIME_SERIES.getName() : IndexMode.STANDARD.getName()); + IndexMetadata.Builder metadataBuilder = IndexMetadata.builder(randomAlphaOfLength(5)); + if (isTimeSeries) { + settingsBuilder.put( + randomBoolean() ? IndexMetadata.INDEX_DIMENSIONS.getKey() : IndexMetadata.INDEX_ROUTING_PATH.getKey(), + "dummy_value" + ); + } + return metadataBuilder.settings(settingsBuilder).build(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java index 3f4b863532233..7a82e30ebc04c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java @@ -67,7 +67,8 @@ private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBu null, Map.of(), true, - XContentMeteringParserDecorator.NOOP + XContentMeteringParserDecorator.NOOP, + null ); LuceneDocument doc = mapperService.documentMapper().parse(source).rootDoc(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index e332de965b6eb..4261e5845f3a8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -431,7 +431,7 @@ protected static SourceToParse source( XContentBuilder builder = JsonXContent.contentBuilder().startObject(); build.accept(builder); builder.endObject(); - return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates); + return new SourceToParse(id, BytesReference.bytes(builder), XContentType.JSON, routing, dynamicTemplates, null); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 49fe9d30239ae..799f0da58f827 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -105,4 +106,9 @@ public LuceneDocument rootDoc() { protected void addDoc(LuceneDocument doc) { throw new UnsupportedOperationException(); } + + @Override + public BytesRef getTsid() { + return null; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 7d44d7c9cec4d..8b4d29396e5d5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2444,7 +2444,7 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) { IndexRouting indexRouting = IndexRouting.fromIndexMetadata(clusterState.metadata().getProject().getIndexSafe(index)); while (true) { String routing = RandomStrings.randomAsciiLettersOfLength(random, 10); - if (shard == indexRouting.indexShard("id", routing, null, null)) { + if (shard == indexRouting.indexShard("id", routing, null, null, null)) { return routing; } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 7c0c1a12b7495..6aa61e2ed38e1 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -23,7 +23,12 @@ public enum FeatureFlag { LOGS_STREAM("es.logs_stream_feature_flag_enabled=true", Version.fromString("9.1.0"), null), PATTERN_TEXT("es.pattern_text_feature_flag_enabled=true", Version.fromString("9.1.0"), null), SYNTHETIC_VECTORS("es.mapping_synthetic_vectors=true", Version.fromString("9.2.0"), null), - RERANK_SNIPPETS("es.text_similarity_reranker_snippets=true", Version.fromString("9.2.0"), null); + RERANK_SNIPPETS("es.text_similarity_reranker_snippets=true", Version.fromString("9.2.0"), null), + INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG( + "es.index_dimensions_tsid_optimization_feature_flag_enabled=true", + Version.fromString("9.2.0"), + null + ); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 6bc25215855ee..beb43b1ee6c9b 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -331,6 +331,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING); replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); + replicatedSettings.add(IndexMetadata.INDEX_DIMENSIONS); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 4ebc121b116f6..a0e8df75a16e9 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -943,6 +943,9 @@ private void createDownsampleIndex( IndexSettings.TIME_SERIES_END_TIME.getKey(), sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_END_TIME.getKey()) ); + if (sourceIndexMetadata.getTimeSeriesDimensions().isEmpty() == false) { + builder.putList(IndexMetadata.INDEX_DIMENSIONS.getKey(), sourceIndexMetadata.getTimeSeriesDimensions()); + } if (sourceIndexMetadata.getSettings().hasValue(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) { builder.put( MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java index 96b3fc0311835..4d4965fb9115e 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java @@ -11,7 +11,9 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -57,12 +59,17 @@ import java.util.Collection; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; public class DownsampleDataStreamTests extends ESSingleNodeTestCase { @@ -76,7 +83,8 @@ protected Collection> getPlugins() { public void testDataStreamDownsample() throws ExecutionException, InterruptedException, IOException { // GIVEN final String dataStreamName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); - putComposableIndexTemplate("1", List.of(dataStreamName)); + boolean manuallyAddedRoutingPath = randomBoolean(); + putComposableIndexTemplate("1", List.of(dataStreamName), manuallyAddedRoutingPath ? List.of("routing_field") : List.of()); client().execute( CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) @@ -184,33 +192,59 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc equalTo(10L) ); }); + + assertResponse(indicesAdmin().getSettings(new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(dataStreamName)), resp -> { + assertThat(resp.getIndexToSettings().keySet(), equalTo(Set.copyOf(backingIndices))); + resp.getIndexToSettings().values().forEach(setting -> { + if (manuallyAddedRoutingPath) { + assertThat(setting.hasValue(IndexMetadata.INDEX_DIMENSIONS.getKey()), equalTo(false)); + assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), containsInAnyOrder("routing_field")); + } else { + if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + assertThat( + setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), + containsInAnyOrder("routing_field", "dimension") + ); + } else { + assertThat(setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), empty()); + } + assertThat( + setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), + containsInAnyOrder("routing_field", "dimension") + ); + } + }); + }); + } - private void putComposableIndexTemplate(final String id, final List patterns) throws IOException { + private void putComposableIndexTemplate(final String id, final List patterns, List routingField) throws IOException { final TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id); - final Template template = new Template( - indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("routing_field")) - .build(), - new CompressedXContent(""" - { - "properties": { - "@timestamp" : { - "type": "date" - }, - "routing_field": { - "type": "keyword", - "time_series_dimension": true - }, - "counter": { - "type": "long", - "time_series_metric": "counter" - } + Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + if (routingField.isEmpty() == false) { + settings.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingField); + } + final Template template = new Template(settings.build(), new CompressedXContent(""" + { + "properties": { + "@timestamp" : { + "type": "date" + }, + "routing_field": { + "type": "keyword", + "time_series_dimension": true + }, + "dimension": { + "type": "keyword", + "time_series_dimension": true + }, + "counter": { + "type": "long", + "time_series_metric": "counter" } } - """), - null - ); + } + """), null); request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(patterns) @@ -219,6 +253,17 @@ private void putComposableIndexTemplate(final String id, final List patt .build() ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + GetComposableIndexTemplateAction.Response getTemplateResponse = client().execute( + GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, id) + ).actionGet(); + ComposableIndexTemplate composableIndexTemplate = getTemplateResponse.indexTemplates().values().iterator().next(); + assertThat( + composableIndexTemplate.template().settings().hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()), + equalTo(routingField.isEmpty() == false) + ); + // the index.dimensions setting will not be present in the template settings, it is added at index creation time + assertThat(composableIndexTemplate.template().settings().get(IndexMetadata.INDEX_DIMENSIONS.getKey()), nullValue()); } private void indexDocs(final String dataStream, int numDocs, long startTime) {