diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java index 5597b87d18aa5..ee3d08b0e02fe 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java @@ -19,6 +19,8 @@ import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup; import org.gradle.api.tasks.Input; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Iterator; @@ -27,6 +29,7 @@ */ public class Skip implements RestTestTransformGlobalSetup, RestTestTransformByParentObject { + private static final Logger log = LoggerFactory.getLogger(Skip.class); private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false); private final String skipReason; @@ -87,8 +90,14 @@ private void addSkip(ArrayNode skipParent) { @Override public void transformTest(ObjectNode parent) { if (testName.isBlank() == false) { - assert parent.get(testName) instanceof ArrayNode; - addSkip((ArrayNode) parent.get(testName)); + JsonNode value = parent.get(testName); + // Only apply skip to test documents where the key is the test name and value is the steps array. + // Do not apply to nested keys with the same name (e.g. "do: get: { ... }" request body). + // This makes it possible to skip tests where the test name is an overloaded term such as + // task.skipTest("tsdb/25_id_generation/delete",...) + if (value instanceof ArrayNode) { + addSkip((ArrayNode) value); + } } } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index eca7caa961afe..8263c7d9dd097 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -134,7 +134,7 @@ public void testTimeRanges() throws Exception { if (randomBoolean()) { templateSettings.put("index.routing_path", "metricset"); } - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos")); @@ -334,7 +334,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { if (randomBoolean()) { settingsBuilder.put("index.routing_path", "metricset"); } - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } request.indexTemplate( @@ -385,7 +385,7 @@ public void testSkippingShards() throws Exception { var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos")); { var templateSettings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var request = new TransportPutComposableIndexTemplateAction.Request("id1"); @@ -454,12 +454,10 @@ public void testTrimId() throws Exception { .put("index.number_of_replicas", 0) // Reduce sync interval to speedup this integraton test, // otherwise by default it will take 30 seconds before minimum retained seqno is updated: - .put("index.soft_deletes.retention_lease.sync_interval", "100ms"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { + .put("index.soft_deletes.retention_lease.sync_interval", "100ms") // This test checks that _id's are pruned, that only applies // when regular _id's are used. - indexSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), false); - } + .put(IndexSettings.SYNTHETIC_ID.getKey(), false); putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName + "*")) @@ -588,7 +586,7 @@ public void testReindexing() throws Exception { String dataStreamName = "my-ds"; String reindexedDataStreamName = "my-reindexed-ds"; var templateSettings = Settings.builder().put("index.mode", "time_series"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); @@ -647,7 +645,7 @@ public void testAddDimensionToMapping() throws Exception { var templateSettings = Settings.builder() .put("index.mode", "time_series") .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabled); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } putTemplateRequest.indexTemplate( @@ -731,7 +729,7 @@ public void testAddDimensionToMapping() throws Exception { public void testDynamicStringDimensions() throws Exception { String dataStreamName = "my-ds"; var templateSettings = Settings.builder().put("index.mode", "time_series"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); @@ -797,7 +795,7 @@ public void testDynamicStringDimensions() throws Exception { public void testDynamicDimensions() throws Exception { String dataStreamName = "my-ds"; var templateSettings = Settings.builder().put("index.mode", "time_series"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index a9d15980df56c..0606f26dd4ab4 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -148,7 +148,7 @@ protected Settings nodeSettings() { public void testIndexingGettingAndSearching() throws Exception { var templateSettings = indexSettings(randomIntBetween(2, 10), 0).put("index.mode", "time_series"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } @@ -225,7 +225,7 @@ public void testIndexingGettingAndSearching() throws Exception { public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { String dataStreamName = "k8s"; var templateSettings = indexSettings(8, 0).put("index.mode", "time_series"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java index 4381b2bfcf611..7c2860ce384f0 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.routing.RecoverySource; @@ -137,7 +138,6 @@ protected Collection> nodePlugins() { } public void testInvalidIndexMode() { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final var indexName = randomIdentifier(); var randomNonTsdbIndexMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.values())); @@ -163,7 +163,6 @@ public void testInvalidIndexMode() { } public void testInvalidCodec() { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final var indexName = randomIdentifier(); internalCluster().startDataOnlyNode(); var randomNonDefaultCodec = randomFrom( @@ -196,8 +195,77 @@ public void testInvalidCodec() { ); } + public void testDisableSyntheticId() throws IOException { + final var indexName = randomIdentifier(); + internalCluster().startDataOnlyNode(); + + Settings settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname") + .put(IndexSettings.SYNTHETIC_ID.getKey(), false) + .build(); + final var mapping = """ + { + "properties": { + "@timestamp": { + "type": "date" + }, + "hostname": { + "type": "keyword", + "time_series_dimension": true + }, + "metric": { + "properties": { + "field": { + "type": "keyword", + "time_series_dimension": true + }, + "value": { + "type": "integer", + "time_series_metric": "counter" + } + } + } + } + } + """; + assertAcked(client().admin().indices().prepareCreate(indexName).setMapping(mapping).setSettings(settings).get()); + + Instant timestamp = Instant.now(); + var results = createDocuments( + indexName, + // t + 0s + document(timestamp, "vm-dev01", "cpu-load", 0), + document(timestamp, "vm-dev02", "cpu-load", 1), + // t + 1s + document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 2), + document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 3), + // t + 0s out-of-order doc + document(timestamp, "vm-dev03", "cpu-load", 4), + // t + 2s + document(timestamp.plus(2, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 5), + document(timestamp.plus(2, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 6), + // t - 1s out-of-order doc + document(timestamp.minus(1, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 7), + // t + 3s + document(timestamp.plus(3, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 8), + document(timestamp.plus(3, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 9) + ); + + // Verify that documents are created + for (var result : results) { + assertThat(result.getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat(result.getVersion(), equalTo(1L)); + } + + flush(indexName); + + var diskUsage = diskUsage(indexName); + var diskUsageIdField = AnalyzeIndexDiskUsageTestUtils.getPerFieldDiskUsage(diskUsage, IdFieldMapper.NAME); + assertThat("_id field should have postings on disk", diskUsageIdField.getInvertedIndexBytes(), greaterThan(0L)); + assertThat("_id field should not have bloom filter usage", diskUsageIdField.getBloomFilterBytes(), equalTo(0L)); + } + public void testSyntheticId() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final boolean useNestedDocs = rarely(); final var dataStreamName = randomIdentifier(); putDataStreamTemplate(dataStreamName, randomIntBetween(1, 5), 0, useNestedDocs); @@ -443,7 +511,6 @@ enum Operation { } public void testGetFromTranslogBySyntheticId() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final boolean useNestedDocs = rarely(); final var dataStreamName = randomIdentifier(); putDataStreamTemplate(dataStreamName, 1, 0, useNestedDocs); @@ -570,7 +637,6 @@ public void testGetFromTranslogBySyntheticId() throws Exception { } public void testRecoveredOperations() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final boolean useNestedDocs = rarely(); // ensure a couple of nodes to have some operations coordinated @@ -815,7 +881,6 @@ enum Operation { } public void testRecoverOperationsFromLocalTranslog() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final boolean useNestedDocs = rarely(); final var dataStreamName = randomIdentifier(); @@ -1123,7 +1188,6 @@ private static void assertTranslogOperation( * Assert that we can still search by synthetic _id after restoring index from snapshot */ public void testCreateSnapshot() throws IOException { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); final boolean useNestedDocs = rarely(); // create index @@ -1243,8 +1307,6 @@ public void testCreateSnapshot() throws IOException { } public void testMerge() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); - final var dataStreamName = randomIdentifier(); putDataStreamTemplate(dataStreamName, 1, 0, rarely()); @@ -1423,8 +1485,10 @@ private static void putDataStreamTemplate( boolean useNestedDocs ) throws IOException { final var settings = indexSettings(primaries, replicas).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) - .put(IndexSettings.SYNTHETIC_ID.getKey(), true); + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1); + if (randomBoolean()) { + settings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); + } if (randomBoolean()) { settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), randomBoolean()); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 5a0333ff88b66..50986cc2d46f0 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.rest.ObjectPath; import org.junit.Before; @@ -40,10 +39,6 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase { - static { - clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG); - } - private static final String COMPONENT_TEMPLATE = """ { "template": { diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index cff9e957eb486..11c762423ad78 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -140,4 +140,16 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> "get/100_synthetic_source/fields with ignore_malformed", "Malformed values are now stored in binary doc values which sort differently than stored fields" ) + task.skipTest("delete/70_tsdb/basic tsdb delete", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/routing_path matches object", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/ids query", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/create operation on top of old document fails over bulk", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/delete over _bulk", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/delete", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/index a new document on top of an old one", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/generates a consistent id", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/create operation on top of old document fails", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/get", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/routing_path matches deep object", "ids have changed after introduction of synthetic id") + task.skipTest("tsdb/25_id_generation/index a new document on top of an old one over bulk", "ids have changed after introduction of synthetic id") }) diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java index ca0b2083c414b..fdd70280bb817 100644 --- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java +++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java @@ -38,7 +38,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .feature(FeatureFlag.SYNTHETIC_VECTORS) .feature(FeatureFlag.RANDOM_SAMPLING) .feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS) - .feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG) .build(); public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index c6f8be82d501f..f70c479dc66ec 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -1,7 +1,7 @@ --- "basic tsdb delete": - skip: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: when cluster has synthetic_id feature use 71_tsdb_synthetic_id_false or 72_tsdb_synthetic_id_true instead - requires: cluster_features: ["gte_v8.13.0"] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/71_tsdb_synthetic_id_false.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/71_tsdb_synthetic_id_false.yml index a73343c833643..d8dbde15b4ff8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/71_tsdb_synthetic_id_false.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/71_tsdb_synthetic_id_false.yml @@ -1,7 +1,7 @@ --- "basic tsdb delete": - requires: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: explicit synthetic_id false only applies when cluster has the feature - requires: cluster_features: ["gte_v8.13.0"] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/72_tsdb_synthetic_id_true.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/72_tsdb_synthetic_id_true.yml index 48dca4be8613c..97aba958621cf 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/72_tsdb_synthetic_id_true.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/72_tsdb_synthetic_id_true.yml @@ -1,7 +1,7 @@ --- "basic tsdb delete": - requires: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: explicit synthetic_id true requires the feature - requires: cluster_features: ["gte_v8.13.0"] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml index 854a863e40b38..3e891ef0baa44 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -4,7 +4,7 @@ setup: reason: "tsdb index with nested field support enabled" - skip: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: when cluster has synthetic_id feature use 161/162_nested_fields_synthetic_id_false/true instead --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml index ddf591a49b4c0..25cffc5bf2575 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml @@ -3,7 +3,7 @@ setup: cluster_features: ["mapper.tsdb_nested_field_support"] reason: "tsdb index with nested field support enabled" - requires: - cluster_features: ["index.time_series_synthetic_id"] + cluster_features: ["index.time_series_synthetic_id_default"] reason: "index.mapping.synthetic_id requires support for synthetic id in the cluster" --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml index d25b42d3940b3..825d62cec8e83 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml @@ -3,7 +3,7 @@ setup: cluster_features: ["mapper.tsdb_nested_field_support"] reason: "tsdb index with nested field support enabled" - requires: - cluster_features: ["index.time_series_synthetic_id"] + cluster_features: ["index.time_series_synthetic_id_default"] reason: "index.mapping.synthetic_id requires support for synthetic id in the cluster" --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 2560dd74b7292..6f6b5fea6196d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -5,7 +5,7 @@ setup: reason: _tsid hashing introduced in 8.13 and tsid routing changed in 8.14 - skip: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: when cluster has synthetic_id feature use 26_id_generation_synthetic_id_false instead - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml index e73c88b0da551..aa17af8654a4b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/26_id_generation_synthetic_id_false.yml @@ -1,7 +1,7 @@ --- setup: - requires: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: explicit synthetic_id false only applies when cluster has the feature - requires: cluster_features: "gte_v8.14.0" @@ -446,6 +446,8 @@ routing_path matches deep object: end_time: 2021-04-29T00:00:00Z number_of_replicas: 0 number_of_shards: 2 + mapping: + synthetic_id: false mappings: properties: "@timestamp": @@ -487,6 +489,8 @@ routing_path matches object: end_time: 2021-04-29T00:00:00Z number_of_replicas: 0 number_of_shards: 2 + mapping: + synthetic_id: false mappings: properties: "@timestamp": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml index 2f5f058a69c01..c62c1b244af75 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/27_id_generation_synthetic_id_true.yml @@ -1,7 +1,7 @@ --- setup: - requires: - cluster_features: "index.time_series_synthetic_id" + cluster_features: "index.time_series_synthetic_id_default" reason: index.mapping.synthetic_id requires the tsdb_synthetic_id feature flag - requires: cluster_features: "gte_v8.14.0" diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 2229663842fa3..55f211624e443 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -47,6 +47,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.search.QueryParserHelper; @@ -2574,15 +2576,12 @@ IndexMetadata build(boolean repair) { String indexModeString = settings.get(IndexSettings.MODE.getKey()); final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null; final boolean isTsdb = indexMode == IndexMode.TIME_SERIES; - boolean useTimeSeriesSyntheticId = false; - if (isTsdb - && IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG - && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { - var setting = settings.get(IndexSettings.SYNTHETIC_ID.getKey()); - if (setting != null && setting.equalsIgnoreCase(Boolean.TRUE.toString())) { - useTimeSeriesSyntheticId = true; - } - } + var syntheticIdSetting = settings.get(IndexSettings.SYNTHETIC_ID.getKey()); + String codecSetting = settings.get(EngineConfig.INDEX_CODEC_SETTING.getKey()); + boolean useTimeSeriesSyntheticId = isTsdb + && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT) + && (codecSetting == null || codecSetting.equalsIgnoreCase(CodecService.DEFAULT_CODEC)) + && (syntheticIdSetting == null || syntheticIdSetting.equalsIgnoreCase(Boolean.TRUE.toString())); final boolean sequenceNumbersDisabled = IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_SEQUENCE_NUMBERS) && settings.getAsBoolean(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false); diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 6de7f0a3aae8b..a170866eb5e92 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -215,6 +215,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.USE_ES_812_POSTINGS_FORMAT, IndexSettings.USE_DOC_VALUES_SKIPPER, IndexSettings.INTRA_MERGE_PARALLELISM_ENABLED_SETTING, + IndexSettings.SYNTHETIC_ID, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { @@ -251,9 +252,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { ) ); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settings.add(IndexSettings.SYNTHETIC_ID); - } if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG) { settings.add(IndexSettings.DISABLE_SEQUENCE_NUMBERS); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java index f46a6d96f1a15..a3609a6854bee 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java @@ -25,6 +25,7 @@ public Set getFeatures() { public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field"); public static final NodeFeature TIME_SERIES_SYNTHETIC_ID = new NodeFeature("index.time_series_synthetic_id"); + public static final NodeFeature TIME_SERIES_SYNTHETIC_ID_DEFAULT = new NodeFeature("index.time_series_synthetic_id_default"); private static final NodeFeature SYNONYMS_SET_LENIENT_ON_NON_EXISTING = new NodeFeature("index.synonyms_set_lenient_on_non_existing"); @@ -49,6 +50,7 @@ public Set getTestFeatures() { return Set.of( LOGSDB_NO_HOST_NAME_FIELD, TIME_SERIES_SYNTHETIC_ID, + TIME_SERIES_SYNTHETIC_ID_DEFAULT, SYNONYMS_SET_LENIENT_ON_NON_EXISTING, THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION, THROW_EXCEPTION_ON_INDEX_CREATION_IF_UNSUPPORTED_VALUE_TYPE_IN_ALIAS, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 639c30299f0fa..a0c9f6d9a2d8e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -681,22 +681,44 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); - public static final boolean TSDB_SYNTHETIC_ID_FEATURE_FLAG = new FeatureFlag("tsdb_synthetic_id").isEnabled(); - public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", false, new Setting.Validator<>() { - @Override - public void validate(Boolean enabled) { - if (enabled) { - if (TSDB_SYNTHETIC_ID_FEATURE_FLAG == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when the feature flag is enabled.", - SYNTHETIC_ID.getKey() - ) - ); - } + /** + * The {@link IndexMode "mode"} of the index. + */ + public static final Setting MODE = Setting.enumSetting( + IndexMode.class, + "index.mode", + IndexMode.STANDARD, + new Setting.Validator<>() { + @Override + public void validate(IndexMode value) {} + + @Override + public void validate(IndexMode value, Map, Object> settings) { + value.validateWithOtherSettings(settings); } - } + + @Override + public Iterator> settings() { + return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); + } + }, + Property.IndexScope, + Property.Final, + Property.ServerlessPublic + ); + + public static final Setting SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", settings -> { + IndexVersion indexVersion = SETTING_INDEX_VERSION_CREATED.get(settings); + IndexMode indexMode = MODE.get(settings); + String codec = INDEX_CODEC_SETTING.get(settings); + return IndexMode.TIME_SERIES.equals(indexMode) + && CodecService.DEFAULT_CODEC.equalsIgnoreCase(codec) + && indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT) + ? Boolean.TRUE.toString() + : Boolean.FALSE.toString(); + }, new Setting.Validator<>() { + @Override + public void validate(Boolean enabled) {} @Override public void validate(Boolean enabled, Map, Object> settings) { @@ -717,7 +739,7 @@ public void validate(Boolean enabled, Map, Object> settings) { } var codecName = (String) settings.get(INDEX_CODEC_SETTING); - if (codecName.equals(CodecService.DEFAULT_CODEC) == false) { + if (codecName.equalsIgnoreCase(CodecService.DEFAULT_CODEC) == false) { throw new IllegalArgumentException( String.format( Locale.ROOT, @@ -731,7 +753,7 @@ public void validate(Boolean enabled, Map, Object> settings) { } var indexVersion = (IndexVersion) settings.get(SETTING_INDEX_VERSION_CREATED); - if (indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94) == false + if (indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT) == false && indexVersion.equals(IndexVersions.ZERO) == false) { // We validate settings in different places before a real indexVersion has been assigned or // is missing for other reasons. In those cases IndexVersion.ZERO is used as fallback value, @@ -742,7 +764,7 @@ public void validate(Boolean enabled, Map, Object> settings) { Locale.ROOT, "The setting [%s] is only permitted for indexVersion [%s] or later. Current indexVersion: [%s].", SYNTHETIC_ID.getKey(), - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, indexVersion ) ); @@ -757,32 +779,6 @@ public Iterator> settings() { } }, Property.IndexScope, Property.Final); - /** - * The {@link IndexMode "mode"} of the index. - */ - public static final Setting MODE = Setting.enumSetting( - IndexMode.class, - "index.mode", - IndexMode.STANDARD, - new Setting.Validator<>() { - @Override - public void validate(IndexMode value) {} - - @Override - public void validate(IndexMode value, Map, Object> settings) { - value.validateWithOtherSettings(settings); - } - - @Override - public Iterator> settings() { - return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); - } - }, - Property.IndexScope, - Property.Final, - Property.ServerlessPublic - ); - public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting("index.mapping.use_doc_values_skipper", s -> { IndexVersion iv = SETTING_INDEX_VERSION_CREATED.get(s); if (MODE.get(s) == IndexMode.TIME_SERIES) { @@ -1369,7 +1365,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti && scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK_SIZE); useEs812PostingsFormat = scopedSettings.get(USE_ES_812_POSTINGS_FORMAT); intraMergeParallelismEnabled = scopedSettings.get(INTRA_MERGE_PARALLELISM_ENABLED_SETTING); - useTimeSeriesSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(SYNTHETIC_ID); + useTimeSeriesSyntheticId = scopedSettings.get(SYNTHETIC_ID); if (indexMetadata.useTimeSeriesSyntheticId() != useTimeSeriesSyntheticId) { throw new IllegalArgumentException( String.format( diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 9ea1badf07cb6..c683a20e5ca8e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -227,6 +227,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion DISABLE_SEQUENCE_NUMBERS = def(9_074_0_00, Version.LUCENE_10_3_2); public static final IndexVersion UPGRADE_TO_LUCENE_10_4_0 = def(9_075_00_0, Version.LUCENE_10_4_0); public static final IndexVersion UPGRADE_DISKBBQ_ES940 = def(9_076_00_0, Version.LUCENE_10_4_0); + public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT = def(9_077_0_00, Version.LUCENE_10_4_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 1756848a48ab4..c9ced5ac58a29 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -664,8 +664,7 @@ public void testRoutingPathObjectArraysInSource() throws IOException { } public void testRoutingPathBwc() throws IOException { - boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean(); - TimeSeriesRoutingFixture fixture = indexRoutingForRoutingPath(IndexVersion.current(), 8, "dim.*,other.*,top", useSyntheticId); + TimeSeriesRoutingFixture fixture = indexRoutingForRoutingPath(IndexVersion.current(), 8, "dim.*,other.*,top", randomBoolean()); /* * These are the expected shards when we first added routing_path. If these values change * time series will be routed to unexpected shards. You may modify @@ -684,12 +683,11 @@ public void testRoutingPathBwc() throws IOException { } public void testRoutingPathBwcAfterTsidBasedRouting() throws IOException { - boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean(); TimeSeriesRoutingFixture fixture = indexRoutingForTimeSeriesDimensions( IndexVersion.current(), 8, "dim.*,other.*,top", - useSyntheticId + randomBoolean() ); /* * These are the expected shards after tsid based routing. If these values change @@ -789,10 +787,8 @@ public void testRerouteToTargetLogsdb() { public void testRerouteToTargetTsid() { int shards = between(2, 500); Settings.Builder settingsBuilder = settings(IndexVersion.current()).put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "top") - .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean()); - } + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean()); IndexMetadata startingMetadata = IndexMetadata.builder("test") .settings(settingsBuilder) .numberOfShards(shards) @@ -1069,7 +1065,7 @@ private TimeSeriesRoutingFixture indexRoutingForPath(int shards, String path) { // old way of routing paths created during routing // current way of routing paths created during routing via tsid String setting = randomBoolean() ? IndexMetadata.INDEX_DIMENSIONS.getKey() : IndexMetadata.INDEX_ROUTING_PATH.getKey(); - boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean(); + boolean useSyntheticId = randomBoolean(); return getIndexRoutingWithSetting(IndexVersion.current(), shards, path, setting, useSyntheticId); } @@ -1102,12 +1098,12 @@ private static TimeSeriesRoutingFixture getIndexRoutingWithSetting( boolean useSyntheticId ) { if (useSyntheticId) { - assert indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94) : "Can't use synthetic id with this index version"; - assert IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG : "Can't use synthetic id without feature flag"; + assert indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT) + : "Can't use synthetic id with this index version"; } Settings.Builder settingsBuilder = settings(indexVersion).put(setting, path) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { + if (indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT)) { settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); } return new TimeSeriesRoutingFixture( diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index a4d74c9b02cd4..088fd4aaaa4ae 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -957,9 +957,8 @@ public void testSame() { } public void testSyntheticIdCorrectSettings() { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, IndexVersion.current() ); IndexMode mode = IndexMode.TIME_SERIES; @@ -979,8 +978,7 @@ public void testSyntheticIdCorrectSettings() { } public void testSyntheticIdBadVersion() { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); - IndexVersion badVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94); + IndexVersion badVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT); IndexMode mode = IndexMode.TIME_SERIES; String codec = CodecService.DEFAULT_CODEC; @@ -1000,7 +998,7 @@ public void testSyntheticIdBadVersion() { Locale.ROOT, "The setting [%s] is only permitted for indexVersion [%s] or later. Current indexVersion: [%s].", IndexSettings.SYNTHETIC_ID.getKey(), - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, badVersion ) ) @@ -1008,9 +1006,8 @@ public void testSyntheticIdBadVersion() { } public void testSyntheticIdBadCodec() { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, IndexVersion.current() ); IndexMode mode = IndexMode.TIME_SERIES; @@ -1046,9 +1043,8 @@ public void testSyntheticIdBadCodec() { } public void testSyntheticIdBadMode() { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); IndexVersion version = IndexVersionUtils.randomVersionBetween( - IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94, + IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT, IndexVersion.current() ); IndexMode badMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.values())); diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 8d31b2e4bf2c2..fb6d0defeeaaa 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -66,7 +66,6 @@ public void testDefault() throws Exception { } public void testTSDBDefault() throws Exception { - assumeTrue("Only when synthetic id feature flag is enabled", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); boolean syntheticIdEnabled = randomBoolean(); CodecService codecService = createCodecService(syntheticIdEnabled); Codec codec = codecService.codec("default"); @@ -144,7 +143,6 @@ private CodecService createCodecService(boolean syntheticIdEnabled) throws IOExc Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); var indexSettings = Settings.builder().put(nodeSettings); if (syntheticIdEnabled) { - assertTrue(IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); indexSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "hostname"); diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index ba87c928cf977..844d1ee95021e 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -222,9 +222,7 @@ private PerFieldFormatSupplier createFormatSupplier( settings.put(IndexSettings.MODE.getKey(), "time_series"); settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settings.put(IndexSettings.SYNTHETIC_ID.getKey(), syntheticId); - } + settings.put(IndexSettings.SYNTHETIC_ID.getKey(), syntheticId); if (disableBloomFilter) { settings.put(IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.getKey(), false); } @@ -332,7 +330,7 @@ private PerFieldFormatSupplier createFormatSupplier( if (mode == IndexMode.TIME_SERIES) { settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && syntheticId != null) { + if (syntheticId != null) { settings.put(IndexSettings.SYNTHETIC_ID.getKey(), syntheticId); } if (enableES87TSDBCodec != null) { @@ -350,6 +348,6 @@ private PerFieldFormatSupplier createFormatSupplier( } private static boolean syntheticId(boolean timeSeries) { - return IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && timeSeries && randomBoolean(); + return timeSeries && randomBoolean(); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/storedfields/TSDBStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/storedfields/TSDBStoredFieldsFormatTests.java index 548d748f95f16..09c725d17f51e 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/storedfields/TSDBStoredFieldsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/storedfields/TSDBStoredFieldsFormatTests.java @@ -16,7 +16,6 @@ import org.apache.lucene.tests.codecs.asserting.AssertingCodec; import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.tsdb.TSDBSyntheticIdPostingsFormatTests; import org.elasticsearch.index.mapper.IdFieldMapper; @@ -42,7 +41,6 @@ public StoredFieldsFormat storedFieldsFormat() { } public void testSyntheticId() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); runTestWithRandomDocs((writer, finalDocs) -> { try (var reader = DirectoryReader.open(writer)) { final var storedFields = reader.storedFields(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdPostingsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdPostingsFormatTests.java index a0ebf980d465a..6867b848f5301 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdPostingsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdPostingsFormatTests.java @@ -87,7 +87,6 @@ public class TSDBSyntheticIdPostingsFormatTests extends ESTestCase { public record Doc(long timestamp, String hostName, String metricField, Integer metricValue, int version, int routing) {} public void testTerms() throws IOException { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); runTest((writer, parser) -> { final var now = Instant.now(); @@ -195,7 +194,6 @@ public void testTerms() throws IOException { } public void testSeek() throws IOException { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); runTestWithRandomDocs((writer, finalDocs) -> { try (var reader = DirectoryReader.open(writer)) { assertThat(reader.getDocCount(IdFieldMapper.NAME), equalTo(finalDocs.values().stream().mapToInt(Doc::version).sum())); diff --git a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java index ccea7e3a4d7c2..0dad58733da4e 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java @@ -719,10 +719,10 @@ public void testWrapForMergeUnwrapsSyntheticIdStoredFieldsReader() throws IOExce private static IndexSettings timeSeriesIndexSettings(boolean useSyntheticId, boolean pruneSequenceNumber) { IndexVersion minVersion; - if (pruneSequenceNumber) { + if (useSyntheticId) { + minVersion = IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT; + } else if (pruneSequenceNumber) { minVersion = IndexVersions.DISABLE_SEQUENCE_NUMBERS; - } else if (useSyntheticId) { - minVersion = IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94; } else { minVersion = IndexVersions.TIME_SERIES_ID_HASHING; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index e91de26e53c3d..c42c030b8dbb6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -140,16 +140,14 @@ public void testStoredValue() throws IOException { public void testDefaultsForTimeSeriesIndex() throws IOException { var isStored = randomBoolean(); - boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean(); + boolean useSyntheticId = randomBoolean(); - var indexSettingsBuilder = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - indexSettingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); - } - var indexSettings = indexSettingsBuilder.build(); + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z") + .put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId) + .build(); var mapping = mapping(b -> { b.startObject("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 2f75eddded7e9..8e5caaaf00535 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -732,11 +732,9 @@ public void testRecoverySourceWithTimeSeries() throws IOException { { Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") // Synthetic _ids are not relevant to this test and the "123" id used is not synthetic-id-conformant - settings.put(IndexSettings.SYNTHETIC_ID.getKey(), false); - } + .put(IndexSettings.SYNTHETIC_ID.getKey(), false); MapperService mapperService = createMapperService(settings.build(), fieldMapping(b -> { b.field("type", "keyword"); b.field("time_series_dimension", true); @@ -749,11 +747,9 @@ public void testRecoverySourceWithTimeSeries() throws IOException { Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") - .put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { + .put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false) // Synthetic _ids are not relevant to this test and the "123" id used is not synthetic-id-conformant - settings.put(IndexSettings.SYNTHETIC_ID.getKey(), false); - } + .put(IndexSettings.SYNTHETIC_ID.getKey(), false); MapperService mapperService = createMapperService(settings.build(), fieldMapping(b -> { b.field("type", "keyword"); b.field("time_series_dimension", true); @@ -785,11 +781,9 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { { Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") // Synthetic _ids are not relevant to this test and the "123" id used is not synthetic-id-conformant - settings.put(IndexSettings.SYNTHETIC_ID.getKey(), false); - } + .put(IndexSettings.SYNTHETIC_ID.getKey(), false); MapperService mapperService = createMapperService(settings.build(), mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse(source("123", b -> b.field("@timestamp", "2012-02-13").field("field", "value1"), null)); @@ -799,11 +793,9 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException { Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field") - .put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { + .put(INDICES_RECOVERY_SOURCE_ENABLED_SETTING.getKey(), false) // Synthetic _ids are not relevant to this test and the "123" id used is not synthetic-id-conformant - settings.put(IndexSettings.SYNTHETIC_ID.getKey(), false); - } + .put(IndexSettings.SYNTHETIC_ID.getKey(), false); MapperService mapperService = createMapperService(settings.build(), mappings); DocumentMapper docMapper = mapperService.documentMapper(); ParsedDocument doc = docMapper.parse( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 82fd83a041531..4325179cab5bc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -2335,17 +2335,14 @@ public void testNormsDisabledWhenIndexModeIsLogsDb() throws IOException { public void testNormsDisabledWhenIndexModeIsTsdb() throws IOException { // given - final boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean(); + final boolean useSyntheticId = randomBoolean(); Instant currentTime = Instant.now(); - Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder(); - indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + Settings indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), currentTime.minus(1, ChronoUnit.HOURS).toEpochMilli()) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), currentTime.plus(1, ChronoUnit.HOURS).toEpochMilli()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - indexSettingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); - } - Settings indexSettings = indexSettingsBuilder.build(); + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension") + .put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId) + .build(); XContentBuilder mapping = mapping(b -> { b.startObject("potato"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index c539cab41c41f..26799b1db3f2f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -991,17 +991,13 @@ private Settings indexSettings(IndexVersion version, boolean indexDimensions, bo } else { builder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "r1,r2,o.r3"); } - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) { + if (version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT)) { builder.put(IndexSettings.SYNTHETIC_ID.getKey(), syntheticId); } return builder.build(); } private MapperService mapperService(boolean indexDimensions, boolean syntheticId, IndexVersion version) throws IOException { - if (syntheticId) { - assumeTrue("Only run with syntheticId if feature flag is enabled", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); - } - return createMapperService(version, indexSettings(version, indexDimensions, syntheticId), mapping(b -> { b.startObject("r1").field("type", "keyword").field("time_series_dimension", true).endObject(); b.startObject("r2").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -1045,7 +1041,9 @@ protected boolean isConfigurable() { protected void registerParameters(ParameterChecker checker) throws IOException {} public void testSourceDescriptionWithRoutingPath() throws IOException { - IndexVersion minimalVersion = useSyntheticId ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94 : IndexVersions.TIME_SERIES_ID_HASHING; + IndexVersion minimalVersion = useSyntheticId + ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT + : IndexVersions.TIME_SERIES_ID_HASHING; IndexVersion version = IndexVersionUtils.randomVersionOnOrAfter(minimalVersion); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(version, false)), @@ -1071,7 +1069,9 @@ public void testSourceDescriptionWithRoutingPath() throws IOException { } public void testSourceDescriptionWithIndexDimensions() throws IOException { - IndexVersion minimalVersion = useSyntheticId ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94 : IndexVersions.TIME_SERIES_ID_HASHING; + IndexVersion minimalVersion = useSyntheticId + ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT + : IndexVersions.TIME_SERIES_ID_HASHING; IndexVersion version = IndexVersionUtils.randomVersionOnOrAfter(minimalVersion); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(version, true)), @@ -1118,7 +1118,7 @@ private TestDocumentParserContext documentParserContext(IndexVersion version, bo public void testParsedDescriptionWithRoutingPath() throws IOException { IndexVersion minimumVersion = useSyntheticId - ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94 + ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT : IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID; IndexVersion version = IndexVersionUtils.randomVersionOnOrAfter(minimumVersion); assertThat( @@ -1140,7 +1140,7 @@ public void testParsedDescriptionWithRoutingPath() throws IOException { public void testParsedDescriptionWithIndexDimensions() throws IOException { // not using a random source here as the index.dimensions id is sensitive to how ips are represented (e.g. equivalent ipv4 vs ipv6) IndexVersion minimumVersion = useSyntheticId - ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94 + ? IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT : IndexVersions.TSID_CREATED_DURING_ROUTING; IndexVersion version = IndexVersionUtils.randomVersionOnOrAfter(minimumVersion); assertThat( diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index e194a8cfd236d..cf01eb5995c13 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -26,7 +26,6 @@ public enum FeatureFlag { ), RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null), - TSDB_SYNTHETIC_ID_FEATURE_FLAG("es.tsdb_synthetic_id_feature_flag_enabled=true", Version.fromString("9.3.0"), null), ESQL_VIEWS("es.esql_views_feature_flag_enabled=true", Version.fromString("9.3.0"), null), EXTENDED_DOC_VALUES_PARAMS("es.extended_doc_values_options_feature_flag_enabled=true", Version.fromString("9.3.0"), null), PROMETHEUS_FEATURE_FLAG("es.prometheus_feature_flag_enabled=true", Version.fromString("9.4.0"), null); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrTimeSeriesDataStreamsIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrTimeSeriesDataStreamsIT.java index e7eb10ca7fe78..ad6441c144321 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrTimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrTimeSeriesDataStreamsIT.java @@ -74,7 +74,6 @@ public void testCrossClusterReplicationForTSDB() throws Exception { } public void testCrossClusterReplicationForTSDBWithSyntheticId() throws Exception { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); executeTest(true); } @@ -276,17 +275,16 @@ enum Operation { private static void putDataStreamTemplate(Client client, String dataStreamName, int primaries, int replicas, boolean useSyntheticId) throws IOException { - var settingsBuilder = indexSettings(primaries, replicas).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) + var settings = indexSettings(primaries, replicas).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) - .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), randomBoolean()); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); - } + .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), randomBoolean()) + .put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId) + .build(); var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request(getTestClass().getName().toLowerCase(Locale.ROOT)) .indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName)) - .template(new Template(settingsBuilder.build(), new CompressedXContent(""" + .template(new Template(settings, new CompressedXContent(""" { "_doc": { "properties": { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index df752384e4134..1e8d69162df5f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -781,16 +781,14 @@ private CheckedBiFunction nestedPa public void testProcessOnceOnPrimary() throws Exception { final Settings.Builder settingsBuilder = indexSettings(IndexVersion.current(), 1, 0).put("index.xpack.ccr.following_index", true); - boolean useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && indexMode == IndexMode.TIME_SERIES && randomBoolean(); + boolean useSyntheticId = indexMode == IndexMode.TIME_SERIES && randomBoolean(); switch (indexMode) { case STANDARD: break; case TIME_SERIES: settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo"); settingsBuilder.put("index.seq_no.index_options", "points_and_doc_values"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); - } + settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), useSyntheticId); break; case LOGSDB: settingsBuilder.put("index.mode", IndexMode.LOGSDB.getName()); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java index 8f830349da87e..efd9ebcc5032e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java @@ -49,7 +49,7 @@ protected String getTestRestCluster() { public void testTimeSeriesQuerying() throws IOException { var settings = Settings.builder() .loadFromStream("tsdb-settings.json", CsvTestsDataLoader.getResourceStream("/tsdb-settings.json"), false); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { settings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } String mapping = CsvTestsDataLoader.getResourceString("/tsdb-k8s-mapping.json"); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/RandomizedTimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/RandomizedTimeSeriesIT.java index 89530177712e9..4e55c14abb8e3 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/RandomizedTimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/RandomizedTimeSeriesIT.java @@ -553,9 +553,7 @@ void putTSDBIndexTemplate(List patterns, @Nullable String mappingString) settingsBuilder.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, ESTestCase.randomIntBetween(1, 5)); settingsBuilder.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2025-07-31T00:00:00Z"); settingsBuilder.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2025-07-31T12:00:00Z"); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean()); - } + settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean()); CompressedXContent mappings = mappingString == null ? null : CompressedXContent.fromJSON(mappingString); TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request( RandomizedTimeSeriesIT.DATASTREAM_NAME diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index be3b947ea2fe6..e434e862e0a47 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -111,7 +111,7 @@ static Double computeRate(List values) { @Before public void populateIndex() { Settings.Builder settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host", "cluster")); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { + if (randomBoolean()) { settings.put("index.codec", "default").put("index.number_of_replicas", 0).put(IndexSettings.SYNTHETIC_ID.getKey(), true); } client().admin() diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TSDBSyntheticIdUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TSDBSyntheticIdUpgradeIT.java index 81d5b2b46a4ae..68170e9a27efc 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TSDBSyntheticIdUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TSDBSyntheticIdUpgradeIT.java @@ -35,7 +35,7 @@ public class TSDBSyntheticIdUpgradeIT extends AbstractLogsdbRollingUpgradeTestCa public void testRollingUpgrade() throws IOException { int numNodes = getCluster().getNumNodes(); boolean isServerless = isServerless(); - boolean oldClusterHasSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID); + boolean oldClusterHasSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID_DEFAULT); // This test upgrade all nodes in the cluster one by one, // and create one new index in-between every upgrade, diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java index 932f44d69a66d..d24a82bbe3069 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java @@ -133,7 +133,7 @@ static String getTemplate(Boolean useSyntheticId) { """; public void testTsdbDataStream() throws Exception { - boolean hasSupport = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID); + boolean hasSupport = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID_DEFAULT); Boolean useSyntheticId = hasSupport ? randomBoolean() : null; String dataStreamName = "k8s"; diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java index 469d4d3fc7fb5..86d5db8aea501 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -import static org.elasticsearch.xpack.logsdb.TsdbIT.formatInstant; import static org.elasticsearch.xpack.logsdb.TsdbIT.getTemplate; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -47,7 +46,7 @@ public void testIndexing() throws Exception { // If cluster has support for synthetic id, randomly set index.mapping.synthetic_id to true/false // If cluster doesn't have support, don't set index.mapping.synthetic_id at all (null), // indicated by oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID)==false - Boolean useSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID) ? randomBoolean() : null; + Boolean useSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID_DEFAULT) ? randomBoolean() : null; String dataStreamName = "k9s"; createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), getTemplate(useSyntheticId)); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsTSDBSyntheticIdIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsTSDBSyntheticIdIntegTests.java index 682f353660a0c..0cd4a37b4e8ee 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsTSDBSyntheticIdIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsTSDBSyntheticIdIntegTests.java @@ -69,25 +69,23 @@ public class SearchableSnapshotsTSDBSyntheticIdIntegTests extends BaseFrozenSear @Override public void setUp() throws Exception { super.setUp(); - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - createRepository(REPOSITORY, FsRepository.TYPE); - assertAcked(syntheticIdIndex(INDEX)); - int initialNumberOfDocuments = scaledRandomIntBetween(20, 2_000); - docIds = indexRandomDocuments(INDEX, initialNumberOfDocuments); - deletedDocIds = deleteRandomDocuments(INDEX, docIds); - numberOfDocuments = docIds.size(); - createSnapshot(REPOSITORY, SNAPSHOT, List.of(INDEX)); - assertAcked(indicesAdmin().prepareDelete(INDEX)); - mountSnapshot( - REPOSITORY, - SNAPSHOT, - INDEX, - MOUNTED_INDEX, - Settings.EMPTY, - randomFrom(MountSearchableSnapshotRequest.Storage.values()) - ); - ensureGreen(MOUNTED_INDEX); - } + createRepository(REPOSITORY, FsRepository.TYPE); + assertAcked(syntheticIdIndex(INDEX)); + int initialNumberOfDocuments = scaledRandomIntBetween(20, 2_000); + docIds = indexRandomDocuments(INDEX, initialNumberOfDocuments); + deletedDocIds = deleteRandomDocuments(INDEX, docIds); + numberOfDocuments = docIds.size(); + createSnapshot(REPOSITORY, SNAPSHOT, List.of(INDEX)); + assertAcked(indicesAdmin().prepareDelete(INDEX)); + mountSnapshot( + REPOSITORY, + SNAPSHOT, + INDEX, + MOUNTED_INDEX, + Settings.EMPTY, + randomFrom(MountSearchableSnapshotRequest.Storage.values()) + ); + ensureGreen(MOUNTED_INDEX); } private CreateIndexRequestBuilder syntheticIdIndex(String indexName) { @@ -154,11 +152,9 @@ private Collection deleteRandomDocuments(String index, Set docId @After @Override public void tearDown() throws Exception { - if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) { - assertAcked(indicesAdmin().prepareDelete("mounted-*")); - assertAcked(clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, REPOSITORY, SNAPSHOT).get()); - assertAcked(clusterAdmin().prepareDeleteRepository(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, REPOSITORY)); - } + assertAcked(indicesAdmin().prepareDelete("mounted-*")); + assertAcked(clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, REPOSITORY, SNAPSHOT).get()); + assertAcked(clusterAdmin().prepareDeleteRepository(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, REPOSITORY)); super.tearDown(); } @@ -167,8 +163,6 @@ public void tearDown() throws Exception { * having them all together in one test cuts execution a lot. */ public void testSearchableSnapshot() throws IOException { - assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); - // Index exists assertTrue("Expected index [" + MOUNTED_INDEX + "] to exist, but did not", indexExists(MOUNTED_INDEX));