From 644777ceca572489773c7dfc014601edc4d353b0 Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Fri, 13 Feb 2026 15:36:02 +0100 Subject: [PATCH 1/7] Enable synthetic_id by default if valid Newly created time_series indices use synthetic id by default. The new IndexVersion, TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, prevent old indices (with older index version) from changing default behavior. `index.mapping.synthetic_id` defaults to true if - Feature flag TSDB_SYNTHETIC_ID_FEATURE_FLAG is enabled - index mode is time_series - index version is TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT or later - codec is default or unset and setting is still allowed to be explicitly set if index version is TIME_SERIES_USE_SYNTHETIC_ID_94 or later. Only the default behavior is controlled by TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT. This default logic is duplicated by IndexMetadata to make sure metadata always agree with Settings. MODE setting moved above SYNTHETIC_ID to allow access. --- .../datastreams/TSDBSyntheticIdsIT.java | 66 +++++++++++++++++++ .../cluster/metadata/IndexMetadata.java | 24 ++++--- .../elasticsearch/index/IndexSettings.java | 65 ++++++++++-------- .../elasticsearch/index/IndexVersions.java | 1 + .../index/IndexSettingsTests.java | 39 +++++++++++ 5 files changed, 158 insertions(+), 37 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 4381b2bfcf611..3d85fa9fc0e53 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -24,10 +24,12 @@ import org.elasticsearch.action.admin.indices.diskusage.TransportAnalyzeIndexDiskUsageAction; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.routing.RecoverySource; @@ -44,6 +46,8 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.codec.storedfields.TSDBStoredFieldsFormat; import org.elasticsearch.index.codec.tsdb.TSDBSyntheticIdStoredFieldsReader; @@ -77,6 +81,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import org.hamcrest.Matchers; import java.io.IOException; import java.nio.file.Path; @@ -1292,6 +1297,67 @@ public void testMerge() throws Exception { assertShardsHaveNoIdStoredFieldValuesOnDisk(indices); } + public void testDefaultSetting() throws Exception { + assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + + String indexName = randomIndexName(); + + // Don't set IndexSettings.SYNTHETIC_ID + Settings.Builder settingsBuilder = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname"); + final var mapping = """ + { + "properties": { + "@timestamp": { + "type": "date" + }, + "hostname": { + "type": "keyword", + "time_series_dimension": true + }, + "metric": { + "properties": { + "field": { + "type": "keyword", + "time_series_dimension": true + }, + "value": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } + } + """; + assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settingsBuilder).setMapping(mapping).get()); + + var timestamp = Instant.now(); + createDocuments( + indexName, + document(timestamp, "vm-dev01", "cpu-load", 0), + document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 1) + ); + flushAndRefresh(indexName); + + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(TEST_REQUEST_TIMEOUT, indexName).get(); + String versionSetting = getSettingsResponse.getSetting(indexName, IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey()); + IndexVersion version = IndexVersion.fromId(Integer.parseInt(versionSetting)); + assertTrue(version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT)); + String syntheticIdSetting = getSettingsResponse.getSetting(indexName, IndexSettings.SYNTHETIC_ID.getKey()); + assertThat(syntheticIdSetting, Matchers.nullValue()); + + var diskUsage = diskUsage(indexName); + var diskUsageIdField = AnalyzeIndexDiskUsageTestUtils.getPerFieldDiskUsage(diskUsage, IdFieldMapper.NAME); + assertThat("_id field should not have postings on disk", diskUsageIdField.getInvertedIndexBytes(), equalTo(0L)); + assertThat("_id field should have bloom filter usage", diskUsageIdField.getBloomFilterBytes(), greaterThan(0L)); + + var indices = new HashSet(); + indices.add(indexName); + assertShardsHaveNoIdStoredFieldValuesOnDisk(indices); + } + private static long documentCount(String dataStreamName) { return indicesAdmin().prepareStats(dataStreamName).setDocs(true).get().getTotal().docs.getCount(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 2229663842fa3..e07b00dd523bb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -47,6 +47,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.search.QueryParserHelper; @@ -2574,15 +2576,7 @@ IndexMetadata build(boolean repair) { String indexModeString = settings.get(IndexSettings.MODE.getKey()); final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null; final boolean isTsdb = indexMode == IndexMode.TIME_SERIES; - boolean useTimeSeriesSyntheticId = false; - if (isTsdb - && IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG - && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { - var setting = settings.get(IndexSettings.SYNTHETIC_ID.getKey()); - if (setting != null && setting.equalsIgnoreCase(Boolean.TRUE.toString())) { - useTimeSeriesSyntheticId = true; - } - } + boolean useTimeSeriesSyntheticId = shouldUseTimeSeriesSyntheticId(isTsdb, indexCreatedVersion, settings); final boolean sequenceNumbersDisabled = IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_SEQUENCE_NUMBERS) && settings.getAsBoolean(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false); @@ -2642,6 +2636,18 @@ IndexMetadata build(boolean repair) { ); } + private static boolean shouldUseTimeSeriesSyntheticId(boolean isTsdb, IndexVersion version, Settings settings) { + String codecSetting = settings.get(EngineConfig.INDEX_CODEC_SETTING.getKey()); + if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG + && isTsdb + && version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94) + && (codecSetting == null || codecSetting.equalsIgnoreCase(CodecService.DEFAULT_CODEC))) { + boolean defaultValue = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); + return settings.getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), defaultValue); + } + return false; + } + @SuppressWarnings("unchecked") public static void toXContent(IndexMetadata indexMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException { Metadata.XContentContext context = Metadata.XContentContext.valueOf( diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 639c30299f0fa..554f69ffc5cb0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -681,8 +681,43 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); + /** + * The {@link IndexMode "mode"} of the index. + */ + public static final Setting MODE = Setting.enumSetting( + IndexMode.class, + "index.mode", + IndexMode.STANDARD, + new Setting.Validator<>() { + @Override + public void validate(IndexMode value) {} + + @Override + public void validate(IndexMode value, Map, Object> settings) { + value.validateWithOtherSettings(settings); + } + + @Override + public Iterator> settings() { + return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); + } + }, + Property.IndexScope, + Property.Final, + Property.ServerlessPublic + ); + public static final boolean TSDB_SYNTHETIC_ID_FEATURE_FLAG = new FeatureFlag("tsdb_synthetic_id").isEnabled(); - public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", false, new Setting.Validator<>() { + public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", settings -> { + IndexVersion indexVersion = SETTING_INDEX_VERSION_CREATED.get(settings); + IndexMode indexMode = MODE.get(settings); + String codec = INDEX_CODEC_SETTING.get(settings); + boolean onByDefault = indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); + return TSDB_SYNTHETIC_ID_FEATURE_FLAG + && IndexMode.TIME_SERIES.equals(indexMode) + && CodecService.DEFAULT_CODEC.equalsIgnoreCase(codec) + && onByDefault ? Boolean.TRUE.toString() : Boolean.FALSE.toString(); + }, new Setting.Validator<>() { @Override public void validate(Boolean enabled) { if (enabled) { @@ -717,7 +752,7 @@ public void validate(Boolean enabled, Map, Object> settings) { } var codecName = (String) settings.get(INDEX_CODEC_SETTING); - if (codecName.equals(CodecService.DEFAULT_CODEC) == false) { + if (codecName.equalsIgnoreCase(CodecService.DEFAULT_CODEC) == false) { throw new IllegalArgumentException( String.format( Locale.ROOT, @@ -757,32 +792,6 @@ public Iterator> settings() { } }, Property.IndexScope, Property.Final); - /** - * The {@link IndexMode "mode"} of the index. - */ - public static final Setting MODE = Setting.enumSetting( - IndexMode.class, - "index.mode", - IndexMode.STANDARD, - new Setting.Validator<>() { - @Override - public void validate(IndexMode value) {} - - @Override - public void validate(IndexMode value, Map, Object> settings) { - value.validateWithOtherSettings(settings); - } - - @Override - public Iterator> settings() { - return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); - } - }, - Property.IndexScope, - Property.Final, - Property.ServerlessPublic - ); - public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting("index.mapping.use_doc_values_skipper", s -> { IndexVersion iv = SETTING_INDEX_VERSION_CREATED.get(s); if (MODE.get(s) == IndexMode.TIME_SERIES) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 9ea1badf07cb6..c683a20e5ca8e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -227,6 +227,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion DISABLE_SEQUENCE_NUMBERS = def(9_074_0_00, Version.LUCENE_10_3_2); public static final IndexVersion UPGRADE_TO_LUCENE_10_4_0 = def(9_075_00_0, Version.LUCENE_10_4_0); public static final IndexVersion UPGRADE_DISKBBQ_ES940 = def(9_076_00_0, Version.LUCENE_10_4_0); + public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT = def(9_077_0_00, Version.LUCENE_10_4_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index a4d74c9b02cd4..48a94d4eeb4b5 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -978,6 +978,45 @@ public void testSyntheticIdCorrectSettings() { assertTrue(indexMetadata.useTimeSeriesSyntheticId()); } + public void testSyntheticIdDefaultValueTrue() { + assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + IndexVersion version = IndexVersionUtils.randomVersionBetween( + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, + IndexVersion.current() + ); + IndexMode mode = IndexMode.TIME_SERIES; + String codec = CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertTrue(indexSettings.useTimeSeriesSyntheticId()); + assertTrue(indexMetadata.useTimeSeriesSyntheticId()); + } + + public void testSyntheticIdDefaultValueFalse() { + assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); + IndexMode mode = IndexMode.TIME_SERIES; + String codec = CodecService.DEFAULT_CODEC; + + Settings settings = Settings.builder() + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec) + .put(IndexSettings.MODE.getKey(), mode) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing") + .build(); + IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version); + + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + assertFalse(indexSettings.useTimeSeriesSyntheticId()); + assertFalse(indexMetadata.useTimeSeriesSyntheticId()); + } + public void testSyntheticIdBadVersion() { assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); IndexVersion badVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94); From fec38916bd6b6f2c5a9c4171bf6533bdf6776f5f Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Wed, 11 Mar 2026 16:15:03 +0100 Subject: [PATCH 2/7] Remove unnecessary time_series_dimension from field --- .../java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 3d85fa9fc0e53..a0d2f8a99b243 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -1319,8 +1319,7 @@ public void testDefaultSetting() throws Exception { "metric": { "properties": { "field": { - "type": "keyword", - "time_series_dimension": true + "type": "keyword" }, "value": { "type": "integer", From f45455baa5ba3508f592b7e7d5922ee6550ef782 Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Wed, 11 Mar 2026 16:27:58 +0100 Subject: [PATCH 3/7] Fix test failure in TSDBSyntheticIdsIT --- .../org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index a0d2f8a99b243..8db5edb4a21ea 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -1302,10 +1302,13 @@ public void testDefaultSetting() throws Exception { String indexName = randomIndexName(); - // Don't set IndexSettings.SYNTHETIC_ID + // Don't set IndexSettings.SYNTHETIC_ID to test default behavior. + // Use default codec so the SYNTHETIC_ID default is true + // (codec will be randomised by ESIntegTestCase.randomIndexTemplate if not explicitly set) Settings.Builder settingsBuilder = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname"); + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname") + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), CodecService.DEFAULT_CODEC); final var mapping = """ { "properties": { From d299af1cfd2ba756d005522be2db7ca2b326e45f Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Thu, 12 Mar 2026 11:03:24 +0100 Subject: [PATCH 4/7] NodeFeature index.time_series_synthetic_id_default Making sure to add this node feature in the same PR as adding the new index version, `TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT` to make sure the node feature can be used as a shortcut / replacement for checking version in tests. --- server/src/main/java/org/elasticsearch/index/IndexFeatures.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java index f073ea61727fc..5f49f3af89582 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java @@ -25,6 +25,7 @@ public Set getFeatures() { public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field"); public static final NodeFeature TIME_SERIES_SYNTHETIC_ID = new NodeFeature("index.time_series_synthetic_id"); + public static final NodeFeature TIME_SERIES_SYNTHETIC_ID_DEFAULT = new NodeFeature("index.time_series_synthetic_id_default"); public static final NodeFeature TIME_SERIES_NO_SEQNO = new NodeFeature("index.time_series_no_seqno"); @@ -51,6 +52,7 @@ public Set getTestFeatures() { return Set.of( LOGSDB_NO_HOST_NAME_FIELD, TIME_SERIES_SYNTHETIC_ID, + TIME_SERIES_SYNTHETIC_ID_DEFAULT, TIME_SERIES_NO_SEQNO, SYNONYMS_SET_LENIENT_ON_NON_EXISTING, THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION, From 8358b29046cf9640499235202a2e3ca7893dedcb Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Tue, 3 Mar 2026 16:16:15 +0100 Subject: [PATCH 5/7] Skip tsdb/25_id... and delete/70_tsdb in yamlRestCompatTest The compat tests run with a copy of the yaml file from the BWC branch, which doesn't contain the skip for index.time_series_synthetic_id. This means the hardcoded ids in the test doesn't match with actual default behavior anymore, so we skip those tests in the compatibility run. This is safe because the behavior is not changed for existing indices that don't use synthetic id. Only new indices will have the changed behavior, so there is no compatibility issue. To make this work we nedd to make it possible to skip compatibility tests with overloaded names. If the name of the test is also a name of an operation such as `get` or `delete` in below examples, then the build would fail with ``` Execution failed for task ':rest-api-spec:yamlRestCompatTestTransform'. > class com.fasterxml.jackson.databind.node.ObjectNode cannot be cast to class com.fasterxml.jackson.databind.node.ArrayNode ``` Instead of throwing an (assert) exception if the node to transform (skip) is of the wrong type, we now just ignore it instead. Example of test that could not be skipped before, but that can be skipped now: ``` get: - do: get: index: id_generation_test id: cZZNs7B9sSWsyrL5AAABeRnSA5M ``` Written together with Cursor --- .../internal/test/rest/transform/skip/Skip.java | 13 +++++++++++-- rest-api-spec/build.gradle | 12 ++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java index 5597b87d18aa5..ee3d08b0e02fe 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java @@ -19,6 +19,8 @@ import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup; import org.gradle.api.tasks.Input; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Iterator; @@ -27,6 +29,7 @@ */ public class Skip implements RestTestTransformGlobalSetup, RestTestTransformByParentObject { + private static final Logger log = LoggerFactory.getLogger(Skip.class); private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false); private final String skipReason; @@ -87,8 +90,14 @@ private void addSkip(ArrayNode skipParent) { @Override public void transformTest(ObjectNode parent) { if (testName.isBlank() == false) { - assert parent.get(testName) instanceof ArrayNode; - addSkip((ArrayNode) parent.get(testName)); + JsonNode value = parent.get(testName); + // Only apply skip to test documents where the key is the test name and value is the steps array. + // Do not apply to nested keys with the same name (e.g. "do: get: { ... }" request body). + // This makes it possible to skip tests where the test name is an overloaded term such as + // task.skipTest("tsdb/25_id_generation/delete",...) + if (value instanceof ArrayNode) { + addSkip((ArrayNode) value); + } } } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index cff9e957eb486..11c762423ad78 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -140,4 +140,16 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> "get/100_synthetic_source/fields with ignore_malformed", "Malformed values are now stored in binary doc values which sort differently than stored fields" ) + task.skipTest("delete/70_tsdb/basic tsdb delete", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/routing_path matches object", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/ids query", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/create operation on top of old document fails over bulk", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/delete over _bulk", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/delete", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/index a new document on top of an old one", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/generates a consistent id", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/create operation on top of old document fails", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/get", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/routing_path matches deep object", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/index a new document on top of an old one over bulk", "ids have changed after introduction of synthetic id") }) From e050b55f333f3a026bab17f4418eca8ee9a706eb Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Thu, 12 Mar 2026 14:08:45 +0100 Subject: [PATCH 6/7] Ensure green to make sure replicas have caught up This is an attempt at avoiding this failure: ``` TSDBSyntheticIdsIT > testDefaultSetting FAILED java.lang.AssertionError: searcher was not warmed up yet for source[assert_no_id_stored_field] ``` If replica hasn't caught up to primary before calling flushAndRefresh the replica would not be refreshed and isWarmedUp would not be set in InternalEngine.ExternalReaderManager.refreshIfNeeded. By calling ensureGreen we make sure that replicate has caught up and that searcher will be warmed up. --- .../java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 8db5edb4a21ea..53739111a3e78 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -1341,6 +1341,7 @@ public void testDefaultSetting() throws Exception { document(timestamp, "vm-dev01", "cpu-load", 0), document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 1) ); + ensureGreen(indexName); flushAndRefresh(indexName); GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(TEST_REQUEST_TIMEOUT, indexName).get(); From 9214cb4f29f35c71fc34781633657bd3a9b87756 Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Mon, 16 Mar 2026 16:13:32 +0100 Subject: [PATCH 7/7] Remove unused logger from Skip.java --- .../gradle/internal/test/rest/transform/skip/Skip.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java index ee3d08b0e02fe..83587effb287a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java @@ -19,8 +19,6 @@ import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup; import org.gradle.api.tasks.Input; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Iterator; @@ -29,7 +27,6 @@ */ public class Skip implements RestTestTransformGlobalSetup, RestTestTransformByParentObject { - private static final Logger log = LoggerFactory.getLogger(Skip.class); private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false); private final String skipReason;