diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java index 5597b87d18aa5..83587effb287a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java @@ -87,8 +87,14 @@ private void addSkip(ArrayNode skipParent) { @Override public void transformTest(ObjectNode parent) { if (testName.isBlank() == false) { - assert parent.get(testName) instanceof ArrayNode; - addSkip((ArrayNode) parent.get(testName)); + JsonNode value = parent.get(testName); + // Only apply skip to test documents where the key is the test name and value is the steps array. + // Do not apply to nested keys with the same name (e.g. "do: get: { ... }" request body). + // This makes it possible to skip tests where the test name is an overloaded term such as + // task.skipTest("tsdb/25_id_generation/delete",...) + if (value instanceof ArrayNode) { + addSkip((ArrayNode) value); + } } } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index ae9942595cf91..8b942bb61f537 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.diskusage.TransportAnalyzeIndexDiskUsageAction; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.support.WriteRequest; @@ -1297,6 +1298,70 @@ public void testMerge() throws Exception { assertShardsHaveNoIdStoredFieldValuesOnDisk(indices); } + public void testDefaultSetting() throws Exception { + assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + + String indexName = randomIndexName(); + + // Don't set IndexSettings.SYNTHETIC_ID to test default behavior. + // Use default codec so the SYNTHETIC_ID default is true + // (codec will be randomised by ESIntegTestCase.randomIndexTemplate if not explicitly set) + Settings.Builder settingsBuilder = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname") + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), CodecService.DEFAULT_CODEC); + final var mapping = """ + { + "properties": { + "@timestamp": { + "type": "date" + }, + "hostname": { + "type": "keyword", + "time_series_dimension": true + }, + "metric": { + "properties": { + "field": { + "type": "keyword" + }, + "value": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } + } + """; + assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settingsBuilder).setMapping(mapping).get()); + + var timestamp = Instant.now(); + createDocuments( + indexName, + document(timestamp, "vm-dev01", "cpu-load", 0), + document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 1) + ); + ensureGreen(indexName); + flushAndRefresh(indexName); + + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(TEST_REQUEST_TIMEOUT, indexName).get(); + String versionSetting = getSettingsResponse.getSetting(indexName, IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey()); + IndexVersion version = IndexVersion.fromId(Integer.parseInt(versionSetting)); + assertTrue(version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT)); + String syntheticIdSetting = getSettingsResponse.getSetting(indexName, IndexSettings.SYNTHETIC_ID.getKey()); + assertThat(syntheticIdSetting, Matchers.nullValue()); + + var diskUsage = diskUsage(indexName); + var diskUsageIdField = AnalyzeIndexDiskUsageTestUtils.getPerFieldDiskUsage(diskUsage, IdFieldMapper.NAME); + assertThat("_id field should not have postings on disk", diskUsageIdField.getInvertedIndexBytes(), equalTo(0L)); + assertThat("_id field should have bloom filter usage", diskUsageIdField.getBloomFilterBytes(), greaterThan(0L)); + + var indices = new HashSet(); + indices.add(indexName); + assertShardsHaveNoIdStoredFieldValuesOnDisk(indices); + } + /** * This test verifies that index with synthetic id cannot be created * if index version is too low. Imagine a mixed cluster where node A has diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 885ee726b64ae..2dfc8554d48c4 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -158,4 +158,16 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> "get/100_synthetic_source/fields with ignore_malformed", "Malformed values are now stored in binary doc values which sort differently than stored fields" ) + task.skipTest("delete/70_tsdb/basic tsdb delete", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/routing_path matches object", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/ids query", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/create operation on top of old document fails over bulk", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/delete over _bulk", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/delete", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/index a new document on top of an old one", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/generates a consistent id", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/create operation on top of old document fails", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/get", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/routing_path matches deep object", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/index a new document on top of an old one over bulk", "ids have changed after introduction of synthetic id") }) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 2229663842fa3..e07b00dd523bb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -47,6 +47,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.search.QueryParserHelper; @@ -2574,15 +2576,7 @@ IndexMetadata build(boolean repair) { String indexModeString = settings.get(IndexSettings.MODE.getKey()); final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null; final boolean isTsdb = indexMode == IndexMode.TIME_SERIES; - boolean useTimeSeriesSyntheticId = false; - if (isTsdb - && IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG - && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { - var setting = settings.get(IndexSettings.SYNTHETIC_ID.getKey()); - if (setting != null && setting.equalsIgnoreCase(Boolean.TRUE.toString())) { - useTimeSeriesSyntheticId = true; - } - } + boolean useTimeSeriesSyntheticId = shouldUseTimeSeriesSyntheticId(isTsdb, indexCreatedVersion, settings); final boolean sequenceNumbersDisabled = IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_SEQUENCE_NUMBERS) && settings.getAsBoolean(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false); @@ -2642,6 +2636,18 @@ IndexMetadata build(boolean repair) { ); } + private static boolean shouldUseTimeSeriesSyntheticId(boolean isTsdb, IndexVersion version, Settings settings) { + String codecSetting = settings.get(EngineConfig.INDEX_CODEC_SETTING.getKey()); + if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG + && isTsdb + && version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94) + && (codecSetting == null || codecSetting.equalsIgnoreCase(CodecService.DEFAULT_CODEC))) { + boolean defaultValue = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); + return settings.getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), defaultValue); + } + return false; + } + @SuppressWarnings("unchecked") public static void toXContent(IndexMetadata indexMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException { Metadata.XContentContext context = Metadata.XContentContext.valueOf( diff --git a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java index f073ea61727fc..5f49f3af89582 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java @@ -25,6 +25,7 @@ public Set getFeatures() { public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field"); public static final NodeFeature TIME_SERIES_SYNTHETIC_ID = new NodeFeature("index.time_series_synthetic_id"); + public static final NodeFeature TIME_SERIES_SYNTHETIC_ID_DEFAULT = new NodeFeature("index.time_series_synthetic_id_default"); public static final NodeFeature TIME_SERIES_NO_SEQNO = new NodeFeature("index.time_series_no_seqno"); @@ -51,6 +52,7 @@ public Set getTestFeatures() { return Set.of( LOGSDB_NO_HOST_NAME_FIELD, TIME_SERIES_SYNTHETIC_ID, + TIME_SERIES_SYNTHETIC_ID_DEFAULT, TIME_SERIES_NO_SEQNO, SYNONYMS_SET_LENIENT_ON_NON_EXISTING, THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 18ccb7f77f251..e20c5af77920d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -681,8 +681,43 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); + /** + * The {@link IndexMode "mode"} of the index. + */ + public static final Setting MODE = Setting.enumSetting( + IndexMode.class, + "index.mode", + IndexMode.STANDARD, + new Setting.Validator<>() { + @Override + public void validate(IndexMode value) {} + + @Override + public void validate(IndexMode value, Map, Object> settings) { + value.validateWithOtherSettings(settings); + } + + @Override + public Iterator> settings() { + return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); + } + }, + Property.IndexScope, + Property.Final, + Property.ServerlessPublic + ); + public static final boolean TSDB_SYNTHETIC_ID_FEATURE_FLAG = new FeatureFlag("tsdb_synthetic_id").isEnabled(); - public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", false, new Setting.Validator<>() { + public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", settings -> { + IndexVersion indexVersion = SETTING_INDEX_VERSION_CREATED.get(settings); + IndexMode indexMode = MODE.get(settings); + String codec = INDEX_CODEC_SETTING.get(settings); + boolean onByDefault = indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); + return TSDB_SYNTHETIC_ID_FEATURE_FLAG + && IndexMode.TIME_SERIES.equals(indexMode) + && CodecService.DEFAULT_CODEC.equalsIgnoreCase(codec) + && onByDefault ? Boolean.TRUE.toString() : Boolean.FALSE.toString(); + }, new Setting.Validator<>() { @Override public void validate(Boolean enabled) { if (enabled) { @@ -717,7 +752,7 @@ public void validate(Boolean enabled, Map, Object> settings) { } var codecName = (String) settings.get(INDEX_CODEC_SETTING); - if (codecName.equals(CodecService.DEFAULT_CODEC) == false) { + if (codecName.equalsIgnoreCase(CodecService.DEFAULT_CODEC) == false) { throw new IllegalArgumentException( String.format( Locale.ROOT, @@ -757,32 +792,6 @@ public Iterator> settings() { } }, Property.IndexScope, Property.Final); - /** - * The {@link IndexMode "mode"} of the index. - */ - public static final Setting MODE = Setting.enumSetting( - IndexMode.class, - "index.mode", - IndexMode.STANDARD, - new Setting.Validator<>() { - @Override - public void validate(IndexMode value) {} - - @Override - public void validate(IndexMode value, Map, Object> settings) { - value.validateWithOtherSettings(settings); - } - - @Override - public Iterator> settings() { - return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); - } - }, - Property.IndexScope, - Property.Final, - Property.ServerlessPublic - ); - public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting("index.mapping.use_doc_values_skipper", s -> { IndexVersion iv = SETTING_INDEX_VERSION_CREATED.get(s); if (MODE.get(s) == IndexMode.TIME_SERIES) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 067cc10e7fb13..89545e477f415 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -237,6 +237,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_DISKBBQ_ES940 = def(9_076_00_0, Version.LUCENE_10_4_0); public static final IndexVersion FLATTENED_FIELD_NO_ROOT_DOC_VALUES = def(9_077_0_00, Version.LUCENE_10_4_0); public static final IndexVersion IGNORED_SOURCE_AS_DOC_VALUES = def(9_078_0_00, Version.LUCENE_10_4_0); + public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT = def(9_079_0_00, Version.LUCENE_10_4_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index a4d74c9b02cd4..48a94d4eeb4b5 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -978,6 +978,45 @@ public void testSyntheticIdCorrectSettings() { assertTrue(indexMetadata.useTimeSeriesSyntheticId()); } + public void testSyntheticIdDefaultValueTrue() { + assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + IndexVersion version = IndexVersionUtils.randomVersionBetween( + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, + IndexVersion.current() + ); + IndexMode mode = IndexMode.TIME_SERIES; + String codec = CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertTrue(indexSettings.useTimeSeriesSyntheticId()); + assertTrue(indexMetadata.useTimeSeriesSyntheticId()); + } + + public void testSyntheticIdDefaultValueFalse() { + assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); + IndexMode mode = IndexMode.TIME_SERIES; + String codec = CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertFalse(indexSettings.useTimeSeriesSyntheticId()); + assertFalse(indexMetadata.useTimeSeriesSyntheticId()); + } + public void testSyntheticIdBadVersion() { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); IndexVersion badVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94);