diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 04a2f84701169..200e7c8785262 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -666,10 +666,11 @@ public void testAddDimensionToMapping() throws Exception { assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDsRequest)); if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), equalTo(List.of("metricset"))); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty()); } else { assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); } - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); // put mapping with k8s.pod.uid as another time series dimension var putMappingRequest = new PutMappingRequest(dataStreamName).source(""" @@ -685,10 +686,35 @@ public void testAddDimensionToMapping() throws Exception { assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), containsInAnyOrder("metricset", "k8s.pod.name")); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), empty()); } else { assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); } - assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), equalTo(List.of("metricset"))); + + // put dynamic template defining time series dimensions + // we don't support index.dimensions in that case + putMappingRequest = new PutMappingRequest(dataStreamName).source(""" + { + "dynamic_templates": [ + { + "labels": { + "path_match": "labels.*", + "mapping": { + "type": "keyword", + "time_series_dimension": true + } + } + } + ] + } + """, XContentType.JSON); + assertAcked(client().execute(TransportPutMappingAction.TYPE, putMappingRequest).actionGet()); + assertThat( + getSetting(dataStreamName, IndexMetadata.INDEX_ROUTING_PATH), + containsInAnyOrder("metricset", "labels.*", "k8s.pod.name") + ); + assertThat(getSetting(dataStreamName, IndexMetadata.INDEX_DIMENSIONS), empty()); indexWithPodNames(dataStreamName, Instant.now(), Map.of(), "dog", "cat"); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index dcee11371cafc..fa0c2f76d8517 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -186,11 +186,12 @@ public void testIndexingGettingAndSearching() throws Exception { // validate index: var getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(index)).actionGet(); - assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]")); if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), equalTo("[attributes.*]")); + assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), nullValue()); } else { assertThat(getIndexResponse.getSettings().get(index).get("index.dimensions"), nullValue()); + assertThat(getIndexResponse.getSettings().get(index).get("index.routing_path"), equalTo("[attributes.*]")); } // validate mapping var mapping = getIndexResponse.mappings().get(index).getSourceAsMap(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index e8e210d6b896a..268ef3455cd41 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; @@ -132,14 +133,19 @@ public void provideAdditionalSettings( dimensions ); if (dimensions.isEmpty() == false) { - if (matchesAllDimensions && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + if (matchesAllDimensions + && INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG + && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING)) { // Only set index.dimensions if the paths in the dimensions list match all potential dimension fields. // This is not the case e.g. if a dynamic template matches by match_mapping_type instead of path_match additionalSettings.putList(INDEX_DIMENSIONS.getKey(), dimensions); + } else { + // For older index versions, or when not all dimension fields can be matched via the dimensions list, + // we fall back to use index.routing_path. + // This is less efficient, because the dimensions need to be hashed twice: + // once to determine the shard during routing, and once to create the tsid during document parsing. + additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), dimensions); } - // always populate index.routing_path, so that routing works for older index versions - // this applies to indices created during a rolling upgrade - additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), dimensions); } } } @@ -167,9 +173,10 @@ public void onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documen && new HashSet<>(indexDimensions).equals(new HashSet<>(newIndexDimensions)) == false; if (matchesAllDimensions == false) { // If the new dimensions don't match all potential dimension fields, we need to unset index.dimensions - // so that index.routing_path is used instead. - // This can happen if a new dynamic template is added to an existing index that matches by mapping type instead of path_match. + // and set index.routing_path instead. + // This can happen if a new dynamic template with time_series_dimension: true is added to an existing index. additionalSettings.putList(INDEX_DIMENSIONS.getKey(), List.of()); + additionalSettings.putList(INDEX_ROUTING_PATH.getKey(), newIndexDimensions); } else if (hasChanges) { additionalSettings.putList(INDEX_DIMENSIONS.getKey(), newIndexDimensions); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 40ed440587c04..d896f6c7afa64 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -21,10 +21,12 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; import org.junit.Before; import java.io.IOException; @@ -48,12 +50,19 @@ public class DataStreamIndexSettingsProviderTests extends ESTestCase { private static final TimeValue DEFAULT_LOOK_AHEAD_TIME = TimeValue.timeValueMinutes(30); // default DataStreamIndexSettingsProvider provider; + private boolean indexDimensionsTsidOptimizationEnabled; + private IndexVersion indexVersion; @Before public void setup() { provider = new DataStreamIndexSettingsProvider( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()) ); + indexVersion = randomBoolean() + ? IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.TSID_CREATED_DURING_ROUTING) + : IndexVersionUtils.randomVersionBetween(random(), IndexVersions.TSID_CREATED_DURING_ROUTING, IndexVersion.current()); + indexDimensionsTsidOptimizationEnabled = INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG + && indexVersion.onOrAfter(IndexVersions.TSID_CREATED_DURING_ROUTING); } public void testGetAdditionalIndexSettings() throws Exception { @@ -101,21 +110,22 @@ public void testGetAdditionalIndexSettings() throws Exception { now, settings, List.of(new CompressedXContent(mapping)), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); + assertThat(result.size(), equalTo(4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + if (indexDimensionsTsidOptimizationEnabled) { assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); } else { + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field3", "field4", "field5", "field6")); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); } } @@ -155,7 +165,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw now, settings, List.of(new CompressedXContent(mapping)), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -229,21 +239,22 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { now, settings, List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3)), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: // (in production the index.mode setting is usually provided in an index or component template) result = builder().put(result).put("index.mode", "time_series").build(); - assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); + assertThat(result.size(), equalTo(4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + if (indexDimensionsTsidOptimizationEnabled) { assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("field1", "field3")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); } else { + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("field1", "field3")); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); } } @@ -263,7 +274,7 @@ public void testGetAdditionalIndexSettingsNoMappings() { now, settings, List.of(), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -292,7 +303,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { now, settings, List.of(new CompressedXContent("{}")), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -321,7 +332,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception { now, settings, List.of(new CompressedXContent("{}")), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -357,7 +368,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce now, settings, List.of(new CompressedXContent("{}")), - IndexVersion.current(), + indexVersion, additionalSettings ); var result = additionalSettings.build(); @@ -397,7 +408,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi now, settings, null, - IndexVersion.current(), + indexVersion, builder() ) ); @@ -426,7 +437,7 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { Instant.ofEpochMilli(1L), settings, null, - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -452,7 +463,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { now, settings, List.of(), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -485,7 +496,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() { Instant.ofEpochMilli(1L), Settings.EMPTY, List.of(), - IndexVersion.current(), + indexVersion, additionalSettings ); Settings result = additionalSettings.build(); @@ -706,14 +717,15 @@ public void testGenerateNonDimensionDynamicTemplate() throws Exception { } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); + assertThat(result.size(), equalTo(4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id")); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + if (indexDimensionsTsidOptimizationEnabled) { assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("host.id")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); } else { + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id")); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); } } @@ -793,14 +805,15 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception { } """; Settings result = generateTsdbSettings(mapping, now); - assertThat(result.size(), equalTo(INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG ? 5 : 4)); + assertThat(result.size(), equalTo(4)); assertThat(IndexSettings.MODE.get(result), equalTo(IndexMode.TIME_SERIES)); assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); - assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); - if (INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG) { + if (indexDimensionsTsidOptimizationEnabled) { assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), containsInAnyOrder("labels.*")); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); } else { + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); } } @@ -943,9 +956,10 @@ public void testAddDynamicTemplate() throws Exception { } """; // we don't support index.dimensions with dynamic templates so we'll unset index.dimensions - Settings result = onUpdateMappings("labels.*", "labels.*", mapping); - assertThat(result.size(), equalTo(1)); + Settings result = onUpdateMappings(null, "labels.*", mapping); + assertThat(result.size(), equalTo(2)); assertThat(IndexMetadata.INDEX_DIMENSIONS.get(result), empty()); + assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("labels.*")); } private Settings generateTsdbSettings(String mapping, Instant now) throws IOException { @@ -962,7 +976,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce now, settings, List.of(new CompressedXContent(mapping)), - IndexVersion.current(), + indexVersion, additionalSettings ); var result = additionalSettings.build(); @@ -976,7 +990,7 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String Settings.Builder currentSettings = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexMetadata.INDEX_DIMENSIONS.getKey(), dimensions) - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java index 4d4965fb9115e..900689ff3de7d 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java @@ -205,13 +205,14 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), containsInAnyOrder("routing_field", "dimension") ); + assertThat(setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), empty()); } else { assertThat(setting.getAsList(IndexMetadata.INDEX_DIMENSIONS.getKey()), empty()); + assertThat( + setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), + containsInAnyOrder("routing_field", "dimension") + ); } - assertThat( - setting.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey()), - containsInAnyOrder("routing_field", "dimension") - ); } }); });