diff --git a/docs/changelog/141525.yaml b/docs/changelog/141525.yaml new file mode 100644 index 0000000000000..aa3de0dddd242 --- /dev/null +++ b/docs/changelog/141525.yaml @@ -0,0 +1,5 @@ +area: TSDB +issues: [] +pr: 141525 +summary: Rolling upgrade test for synthetic id +type: enhancement diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index abe50de976d7f..751b78e90d457 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -135,7 +135,7 @@ public void testTimeRanges() throws Exception { templateSettings.put("index.routing_path", "metricset"); } if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos")); @@ -335,7 +335,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { settingsBuilder.put("index.routing_path", "metricset"); } if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - settingsBuilder.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } request.indexTemplate( ComposableIndexTemplate.builder() @@ -386,7 +386,7 @@ public void testSkippingShards() throws Exception { { var templateSettings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset"); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var request = new TransportPutComposableIndexTemplateAction.Request("id1"); request.indexTemplate( @@ -590,7 +590,7 @@ public void testReindexing() throws Exception { String reindexedDataStreamName = "my-reindexed-ds"; var templateSettings = Settings.builder().put("index.mode", "time_series"); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( @@ -649,7 +649,7 @@ public void testAddDimensionToMapping() throws Exception { .put("index.mode", "time_series") .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabled); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() @@ -733,7 +733,7 @@ public void testDynamicStringDimensions() throws Exception { String dataStreamName = "my-ds"; var templateSettings = Settings.builder().put("index.mode", "time_series"); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( @@ -799,7 +799,7 @@ public void testDynamicDimensions() throws Exception { String dataStreamName = "my-ds"; var templateSettings = Settings.builder().put("index.mode", "time_series"); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 24b8a6d1a7442..a9d15980df56c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -149,7 +149,7 @@ protected Settings nodeSettings() { public void testIndexingGettingAndSearching() throws Exception { var templateSettings = indexSettings(randomIntBetween(2, 10), 0).put("index.mode", "time_series"); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var request = new TransportPutComposableIndexTemplateAction.Request("id"); @@ -226,7 +226,7 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { String dataStreamName = "k8s"; var templateSettings = indexSettings(8, 0).put("index.mode", "time_series"); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - templateSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var request = new TransportPutComposableIndexTemplateAction.Request("id"); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index a5cd2d53c5449..aca0c898a4084 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -132,7 +132,7 @@ public void testInvalidIndexMode() { () -> createIndex( indexName, indexSettings(1, 0).put(IndexSettings.MODE.getKey(), randomNonTsdbIndexMode) - .put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true) + .put(IndexSettings.SYNTHETIC_ID.getKey(), true) .build() ) ); @@ -140,7 +140,7 @@ public void testInvalidIndexMode() { exception.getMessage(), containsString( "The setting [" - + IndexSettings.USE_SYNTHETIC_ID.getKey() + + IndexSettings.SYNTHETIC_ID.getKey() + "] is only permitted when [index.mode] is set to [TIME_SERIES]. Current mode: [" + randomNonTsdbIndexMode.getName().toUpperCase(Locale.ROOT) + "]." @@ -165,7 +165,7 @@ public void testInvalidCodec() { indexName, indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "hostname") - .put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true) + .put(IndexSettings.SYNTHETIC_ID.getKey(), true) .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomNonDefaultCodec) .build() ) @@ -174,7 +174,7 @@ public void testInvalidCodec() { exception.getMessage(), containsString( "The setting [" - + IndexSettings.USE_SYNTHETIC_ID.getKey() + + IndexSettings.SYNTHETIC_ID.getKey() + "] is only permitted when [index.codec] is set to [default]. Current mode: [" + randomNonDefaultCodec + "]." @@ -1211,7 +1211,7 @@ private static void putDataStreamTemplate(String indexPattern, int primaries, in private static void putDataStreamTemplate(String indexPattern, int primaries, int replicas, Settings extraSettings) throws IOException { final var settings = indexSettings(primaries, replicas).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) - .put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + .put(IndexSettings.SYNTHETIC_ID.getKey(), true); if (randomBoolean()) { settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), randomBoolean()); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 49494bb62b67b..5a0333ff88b66 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -28,7 +28,7 @@ import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; -import static org.elasticsearch.index.IndexSettings.USE_SYNTHETIC_ID; +import static org.elasticsearch.index.IndexSettings.SYNTHETIC_ID; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -225,17 +225,17 @@ private static String getTemplate() { idMode = switch (randomInt(2)) { case 0 -> null; case 1 -> """ - "use_synthetic_id": "false" + "synthetic_id": "false" """; case 2 -> """ - "use_synthetic_id": "true" + "synthetic_id": "true" """; default -> throw new AssertionError("Unknown mode"); }; } else { assertFalse( "Setting is enabled by default and must now be tested on non-snapshot build too ", - USE_SYNTHETIC_ID.getDefault(Settings.EMPTY) + SYNTHETIC_ID.getDefault(Settings.EMPTY) ); } if (sourceMode == null && idMode == null) { diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TSDBSyntheticIdUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TSDBSyntheticIdUpgradeIT.java new file mode 100644 index 0000000000000..706762b97f96b --- /dev/null +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TSDBSyntheticIdUpgradeIT.java @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.test.rest.ObjectPath; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Locale; +import java.util.Map; +import java.util.StringJoiner; + +public class TSDBSyntheticIdUpgradeIT extends AbstractRollingUpgradeTestCase { + private static final int DOC_COUNT = 10; + + public TSDBSyntheticIdUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + public void testRollingUpgrade() throws IOException { + IndexVersion oldClusterIndexVersion = getOldClusterIndexVersion(); + + if (hasSupportForSyntheticId(oldClusterIndexVersion)) { + if (isOldCluster()) { + assertWriteIndex("old-cluster-index"); + assertIndexRead("old-cluster-index"); + } + + if (isFirstMixedCluster()) { + assertWriteIndex("first-mixed-cluster-index"); + assertIndexRead("old-cluster-index"); + assertIndexRead("first-mixed-cluster-index"); + } + + if (isFirstMixedCluster() == false && isMixedCluster()) { + assertWriteIndex("second-mixed-cluster-index"); + assertIndexRead("old-cluster-index"); + assertIndexRead("first-mixed-cluster-index"); + assertIndexRead("second-mixed-cluster-index"); + } + + if (isUpgradedCluster()) { + assertWriteIndex("upgraded-cluster-index"); + assertIndexRead("old-cluster-index"); + assertIndexRead("first-mixed-cluster-index"); + assertIndexRead("second-mixed-cluster-index"); + assertIndexRead("upgraded-cluster-index"); + } + } else { + + if (isOldCluster()) { + assertNoWriteIndex("old-cluster-index"); + } + + if (isMixedCluster()) { + assertNoWriteIndex("mixed-cluster-index"); + } + + if (isUpgradedCluster()) { + assertWriteIndex("upgraded-cluster-index"); + assertIndexRead("upgraded-cluster-index"); + } + } + } + + private static void assertWriteIndex(String indexName) throws IOException { + assertIndexCanBeCreated(indexName); + assertCanAddDocuments(indexName); + } + + private static void assertIndexRead(String indexName) throws IOException { + assertTrue("Expected index [" + indexName + "] to exist, but did not", indexExists(indexName)); + Map indexSettingsAsMap = getIndexSettingsAsMap(indexName); + assertThat(indexSettingsAsMap.get(IndexSettings.SYNTHETIC_ID.getKey()), Matchers.equalTo("true")); + assertDocCount(client(), indexName, DOC_COUNT); + assertThat(invertedIndexSize(indexName), Matchers.equalTo(0)); + } + + private static int invertedIndexSize(String indexName) throws IOException { + var diskUsage = new Request("POST", "/" + indexName + "/_disk_usage?run_expensive_tasks=true"); + Response response = client().performRequest(diskUsage); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + return objectPath.evaluate(indexName + ".all_fields.inverted_index.total_in_bytes"); + } + + private static void assertIndexCanBeCreated(String indexName) throws IOException { + CreateIndexResponse response = null; + try { + response = createSyntheticIdIndex(indexName); + assertTrue("Expected index [" + indexName + "] to be created successfully, but was not", response.isAcknowledged()); + assertTrue( + "Expected shards of index [" + indexName + "] to be created successfully, but was not", + response.isShardsAcknowledged() + ); + } finally { + if (response != null) { + response.decRef(); + } + } + } + + private static void assertCanAddDocuments(String indexName) throws IOException { + StringJoiner joiner = new StringJoiner("\n", "", "\n"); + Instant now = Instant.now(); + for (int i = 0; i < DOC_COUNT; i++) { + addDocument(joiner, now.plus(i, ChronoUnit.SECONDS)); + } + var request = new Request("PUT", "/" + indexName + "/_bulk"); + request.setJsonEntity(joiner.toString()); + request.addParameter("refresh", "true"); + Response response = client().performRequest(request); + assertOK(response); + } + + private static void addDocument(StringJoiner joiner, Instant timestamp) { + joiner.add("{\"create\": {}}"); + joiner.add(String.format(Locale.ROOT, """ + {"@timestamp": "%s", "hostname": "host", "metric": {"field": "cpu-load", "value": %d}} + """, timestamp, randomByte())); + } + + private static void assertNoWriteIndex(String indexName) { + String setting = IndexSettings.SYNTHETIC_ID.getKey(); + String unknownSetting = "unknown setting [" + setting + "]"; + String versionTooLow = "The setting [" + setting + "] is set to [true] but index metadata has a different value [false]"; + + ResponseException e = assertThrows(ResponseException.class, () -> createSyntheticIdIndex(indexName)); + assertThat(e.getMessage(), Matchers.either(Matchers.containsString(unknownSetting)).or(Matchers.containsString(versionTooLow))); + assertThat(e.getMessage(), Matchers.containsString("illegal_argument_exception")); + } + + private static CreateIndexResponse createSyntheticIdIndex(String indexName) throws IOException { + Settings settings = Settings.builder() + .put(IndexSettings.SYNTHETIC_ID.getKey(), true) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname") + .build(); + final var mapping = """ + { + "properties": { + "@timestamp": { + "type": "date" + }, + "hostname": { + "type": "keyword", + "time_series_dimension": true + }, + "metric": { + "properties": { + "field": { + "type": "keyword", + "time_series_dimension": true + }, + "value": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } + } + """; + return createIndex(indexName, settings, mapping); + } + + private boolean hasSupportForSyntheticId(IndexVersion indexVersion) { + return indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index b4abc4d99f60c..eede0f892fa1d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2537,8 +2537,8 @@ IndexMetadata build(boolean repair) { boolean useTimeSeriesSyntheticId = false; if (isTsdb && IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG - && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID)) { - var setting = settings.get(IndexSettings.USE_SYNTHETIC_ID.getKey()); + && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { + var setting = settings.get(IndexSettings.SYNTHETIC_ID.getKey()); if (setting != null && setting.equalsIgnoreCase(Boolean.TRUE.toString())) { assert IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG; useTimeSeriesSyntheticId = true; diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 3630e040dd03f..5fd04e68a6aa4 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -251,7 +251,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { ); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settings.add(IndexSettings.USE_SYNTHETIC_ID); + settings.add(IndexSettings.SYNTHETIC_ID); } settings.add(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING); BUILT_IN_INDEX_SETTINGS = Collections.unmodifiableSet(settings); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 8a8618b6e79db..01262301a23db 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -682,68 +682,62 @@ public boolean isES87TSDBCodecEnabled() { ); public static final boolean TSDB_SYNTHETIC_ID_FEATURE_FLAG = new FeatureFlag("tsdb_synthetic_id").isEnabled(); - public static final Setting USE_SYNTHETIC_ID = Setting.boolSetting( - "index.mapping.use_synthetic_id", - false, - new Setting.Validator<>() { - @Override - public void validate(Boolean enabled) { - if (enabled) { - if (TSDB_SYNTHETIC_ID_FEATURE_FLAG == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when the feature flag is enabled.", - USE_SYNTHETIC_ID.getKey() - ) - ); - } + public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", false, new Setting.Validator<>() { + @Override + public void validate(Boolean enabled) { + if (enabled) { + if (TSDB_SYNTHETIC_ID_FEATURE_FLAG == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted when the feature flag is enabled.", + SYNTHETIC_ID.getKey() + ) + ); } } + } - @Override - public void validate(Boolean enabled, Map, Object> settings) { - if (enabled) { - // Verify if index mode is TIME_SERIES - var indexMode = (IndexMode) settings.get(MODE); - if (indexMode != IndexMode.TIME_SERIES) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", - USE_SYNTHETIC_ID.getKey(), - MODE.getKey(), - IndexMode.TIME_SERIES.name(), - indexMode.name() - ) - ); - } + @Override + public void validate(Boolean enabled, Map, Object> settings) { + if (enabled) { + // Verify if index mode is TIME_SERIES + var indexMode = (IndexMode) settings.get(MODE); + if (indexMode != IndexMode.TIME_SERIES) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + SYNTHETIC_ID.getKey(), + MODE.getKey(), + IndexMode.TIME_SERIES.name(), + indexMode.name() + ) + ); + } - var codecName = (String) settings.get(INDEX_CODEC_SETTING); - if (codecName.equals(CodecService.DEFAULT_CODEC) == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", - USE_SYNTHETIC_ID.getKey(), - INDEX_CODEC_SETTING.getKey(), - CodecService.DEFAULT_CODEC, - codecName - ) - ); - } + var codecName = (String) settings.get(INDEX_CODEC_SETTING); + if (codecName.equals(CodecService.DEFAULT_CODEC) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + SYNTHETIC_ID.getKey(), + INDEX_CODEC_SETTING.getKey(), + CodecService.DEFAULT_CODEC, + codecName + ) + ); } } + } - @Override - public Iterator> settings() { - List> list = List.of(MODE, INDEX_CODEC_SETTING); - return list.iterator(); - } - }, - Property.IndexScope, - Property.Final - ); + @Override + public Iterator> settings() { + List> list = List.of(MODE, INDEX_CODEC_SETTING); + return list.iterator(); + } + }, Property.IndexScope, Property.Final); /** * The {@link IndexMode "mode"} of the index. @@ -1274,14 +1268,13 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti useTimeSeriesDocValuesFormatLargeBlockSize = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE); useEs812PostingsFormat = scopedSettings.get(USE_ES_812_POSTINGS_FORMAT); intraMergeParallelismEnabled = scopedSettings.get(INTRA_MERGE_PARALLELISM_ENABLED_SETTING); - final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID); + final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(SYNTHETIC_ID); if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) { - assert false; throw new IllegalArgumentException( String.format( Locale.ROOT, "The setting [%s] is set to [%s] but index metadata has a different value [%s].", - USE_SYNTHETIC_ID.getKey(), + SYNTHETIC_ID.getKey(), useSyntheticId, indexMetadata.useTimeSeriesSyntheticId() ) @@ -1291,7 +1284,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti assert TSDB_SYNTHETIC_ID_FEATURE_FLAG; assert indexMetadata.useTimeSeriesSyntheticId(); assert indexMetadata.getIndexMode() == IndexMode.TIME_SERIES : indexMetadata.getIndexMode(); - assert indexMetadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID); + assert indexMetadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94); useTimeSeriesSyntheticId = true; } else { useTimeSeriesSyntheticId = false; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index d3b908ae63be9..ec1db8439ad95 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -221,6 +221,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion STORE_PATTERN_TEXT_FIELDS_IN_BINARY_DOC_VALUES = def(9_068_0_00, Version.LUCENE_10_3_2); public static final IndexVersion DISK_BBQ_QUANTIZE_BITS = def(9_069_0_00, Version.LUCENE_10_3_2); public static final IndexVersion ID_FIELD_USE_ES812_POSTINGS_FORMAT = def(9_070_0_00, Version.LUCENE_10_3_2); + public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID_94 = def(9_071_0_00, Version.LUCENE_10_3_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index 85b0478a55779..9c9a3682225c8 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -51,9 +51,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays, boolean useSyntheticId = mapperService != null && mapperService.getIndexSettings().useTimeSeriesSyntheticId() - && mapperService.getIndexSettings() - .getIndexVersionCreated() - .onOrAfter(IndexVersions.TIME_SERIES_USE_STORED_FIELDS_BLOOM_FILTER_FOR_ID); + && mapperService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94); var legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene103Codec.Mode.BEST_SPEED, mapperService, bigArrays, threadPool); if (useSyntheticId) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 6c73acb0e155f..5d2787b83bf70 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -146,7 +146,7 @@ private PostingsFormat internalGetPostingsFormatForField(String field) { return completionPostingsFormat; } if (mapper instanceof IdFieldMapper - && IndexVersions.ID_FIELD_USE_ES812_POSTINGS_FORMAT.onOrAfter(mapperService.getIndexSettings().getIndexVersionCreated())) { + && mapperService.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.ID_FIELD_USE_ES812_POSTINGS_FORMAT)) { // The default posting format doesn't handle randomly generated IDs well during merging. Several cases have been reported // where a single merge thread uses disproportionate jvm heap memory just for Lucene103BlockTreeTermsWriter.TermsWriter. return es812PostingsFormat; diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 1eab6ec996af1..fdc6795c925ec 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -145,7 +145,7 @@ private CodecService createCodecService(boolean syntheticIdEnabled) throws IOExc var indexSettings = Settings.builder().put(nodeSettings); if (syntheticIdEnabled) { assertTrue(IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); - indexSettings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true) + indexSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "hostname"); } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java index aef961bb6a02b..951e19c5adc11 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java @@ -50,7 +50,7 @@ public void testTimeSeriesQuerying() throws IOException { var settings = Settings.builder() .loadFromStream("tsdb-settings.json", TSDBRestEsqlIT.class.getResourceAsStream("/tsdb-settings.json"), false); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - settings.put(IndexSettings.USE_SYNTHETIC_ID.getKey(), true); + settings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } String mapping = CsvTestsDataLoader.readTextFile(TSDBRestEsqlIT.class.getResource("/tsdb-k8s-mapping.json")); assertTrue("Failed to create index [k8s]", createIndex("k8s", settings.build(), mapping).isAcknowledged()); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 4da9bd82e87f2..1d25a32e5f436 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -112,7 +112,7 @@ static Double computeRate(List values) { public void populateIndex() { Settings.Builder settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host", "cluster")); if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { - settings.put("index.codec", "default").put("index.number_of_replicas", 0).put("index.mapping.use_synthetic_id", true); + settings.put("index.codec", "default").put("index.number_of_replicas", 0).put(IndexSettings.SYNTHETIC_ID.getKey(), true); } client().admin() .indices()