Enable synthetic id feature, remove feature flag

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java

@@ -19,6 +19,8 @@
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup;
 import org.gradle.api.tasks.Input;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Iterator;
 
@@ -27,6 +29,7 @@
  */
 public class Skip implements RestTestTransformGlobalSetup, RestTestTransformByParentObject {
 
+    private static final Logger log = LoggerFactory.getLogger(Skip.class);
     private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false);
 
     private final String skipReason;
@@ -87,8 +90,14 @@ private void addSkip(ArrayNode skipParent) {
     @Override
     public void transformTest(ObjectNode parent) {
         if (testName.isBlank() == false) {
-            assert parent.get(testName) instanceof ArrayNode;
-            addSkip((ArrayNode) parent.get(testName));
+            JsonNode value = parent.get(testName);
+            // Only apply skip to test documents where the key is the test name and value is the steps array.
+            // Do not apply to nested keys with the same name (e.g. "do: get: { ... }" request body).
+            // This makes it possible to skip tests where the test name is an overloaded term such as
+            // task.skipTest("tsdb/25_id_generation/delete",...)
+            if (value instanceof ArrayNode) {
+                addSkip((ArrayNode) value);
+            }
         }
     }
 

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -134,7 +134,7 @@ public void testTimeRanges() throws Exception {
         if (randomBoolean()) {
             templateSettings.put("index.routing_path", "metricset");
         }
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
         var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos"));
@@ -334,7 +334,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception {
         if (randomBoolean()) {
             settingsBuilder.put("index.routing_path", "metricset");
         }
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
         request.indexTemplate(
@@ -385,7 +385,7 @@ public void testSkippingShards() throws Exception {
         var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos"));
         {
             var templateSettings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset");
-            if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+            if (randomBoolean()) {
                 templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
             }
             var request = new TransportPutComposableIndexTemplateAction.Request("id1");
@@ -454,12 +454,10 @@ public void testTrimId() throws Exception {
             .put("index.number_of_replicas", 0)
             // Reduce sync interval to speedup this integraton test,
             // otherwise by default it will take 30 seconds before minimum retained seqno is updated:
-            .put("index.soft_deletes.retention_lease.sync_interval", "100ms");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) {
+            .put("index.soft_deletes.retention_lease.sync_interval", "100ms")
             // This test checks that _id's are pruned, that only applies
             // when regular _id's are used.
-            indexSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), false);
-        }
+            .put(IndexSettings.SYNTHETIC_ID.getKey(), false);
         putTemplateRequest.indexTemplate(
             ComposableIndexTemplate.builder()
                 .indexPatterns(List.of(dataStreamName + "*"))
@@ -588,7 +586,7 @@ public void testReindexing() throws Exception {
         String dataStreamName = "my-ds";
         String reindexedDataStreamName = "my-reindexed-ds";
         var templateSettings = Settings.builder().put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
         var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
@@ -647,7 +645,7 @@ public void testAddDimensionToMapping() throws Exception {
         var templateSettings = Settings.builder()
             .put("index.mode", "time_series")
             .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabled);
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
         putTemplateRequest.indexTemplate(
@@ -731,7 +729,7 @@ public void testAddDimensionToMapping() throws Exception {
     public void testDynamicStringDimensions() throws Exception {
         String dataStreamName = "my-ds";
         var templateSettings = Settings.builder().put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
         var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
@@ -797,7 +795,7 @@ public void testDynamicStringDimensions() throws Exception {
     public void testDynamicDimensions() throws Exception {
         String dataStreamName = "my-ds";
         var templateSettings = Settings.builder().put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
         var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

@@ -148,7 +148,7 @@ protected Settings nodeSettings() {
 
     public void testIndexingGettingAndSearching() throws Exception {
         var templateSettings = indexSettings(randomIntBetween(2, 10), 0).put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
 
@@ -225,7 +225,7 @@ public void testIndexingGettingAndSearching() throws Exception {
     public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {
         String dataStreamName = "k8s";
         var templateSettings = indexSettings(8, 0).put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
+        if (randomBoolean()) {
             templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
         }
 

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -137,7 +138,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testInvalidIndexMode() {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final var indexName = randomIdentifier();
         var randomNonTsdbIndexMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.values()));
 
@@ -163,7 +163,6 @@ public void testInvalidIndexMode() {
     }
 
     public void testInvalidCodec() {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final var indexName = randomIdentifier();
         internalCluster().startDataOnlyNode();
         var randomNonDefaultCodec = randomFrom(
@@ -196,8 +195,77 @@ public void testInvalidCodec() {
         );
     }
 
+    public void testDisableSyntheticId() throws IOException {
+        final var indexName = randomIdentifier();
+        internalCluster().startDataOnlyNode();
+
+        Settings settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname")
+            .put(IndexSettings.SYNTHETIC_ID.getKey(), false)
+            .build();
+        final var mapping = """
+            {
+                "properties": {
+                    "@timestamp": {
+                        "type": "date"
+                    },
+                    "hostname": {
+                        "type": "keyword",
+                        "time_series_dimension": true
+                    },
+                    "metric": {
+                        "properties": {
+                            "field": {
+                                "type": "keyword",
+                                "time_series_dimension": true
+                            },
+                            "value": {
+                                "type": "integer",
+                                "time_series_metric": "counter"
+                            }
+                        }
+                    }
+                }
+            }
+            """;
+        assertAcked(client().admin().indices().prepareCreate(indexName).setMapping(mapping).setSettings(settings).get());
+
+        Instant timestamp = Instant.now();
+        var results = createDocuments(
+            indexName,
+            // t + 0s
+            document(timestamp, "vm-dev01", "cpu-load", 0),
+            document(timestamp, "vm-dev02", "cpu-load", 1),
+            // t + 1s
+            document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 2),
+            document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 3),
+            // t + 0s out-of-order doc
+            document(timestamp, "vm-dev03", "cpu-load", 4),
+            // t + 2s
+            document(timestamp.plus(2, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 5),
+            document(timestamp.plus(2, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 6),
+            // t - 1s out-of-order doc
+            document(timestamp.minus(1, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 7),
+            // t + 3s
+            document(timestamp.plus(3, ChronoUnit.SECONDS), "vm-dev01", "cpu-load", 8),
+            document(timestamp.plus(3, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 9)
+        );
+
+        // Verify that documents are created
+        for (var result : results) {
+            assertThat(result.getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED));
+            assertThat(result.getVersion(), equalTo(1L));
+        }
+
+        flush(indexName);
+
+        var diskUsage = diskUsage(indexName);
+        var diskUsageIdField = AnalyzeIndexDiskUsageTestUtils.getPerFieldDiskUsage(diskUsage, IdFieldMapper.NAME);
+        assertThat("_id field should have postings on disk", diskUsageIdField.getInvertedIndexBytes(), greaterThan(0L));
+        assertThat("_id field should not have bloom filter usage", diskUsageIdField.getBloomFilterBytes(), equalTo(0L));
+    }
+
     public void testSyntheticId() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, randomIntBetween(1, 5), 0, useNestedDocs);
@@ -443,7 +511,6 @@ enum Operation {
     }
 
     public void testGetFromTranslogBySyntheticId() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, 1, 0, useNestedDocs);
@@ -570,7 +637,6 @@ public void testGetFromTranslogBySyntheticId() throws Exception {
     }
 
     public void testRecoveredOperations() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
 
         // ensure a couple of nodes to have some operations coordinated
@@ -815,7 +881,6 @@ enum Operation {
     }
 
     public void testRecoverOperationsFromLocalTranslog() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
 
         final var dataStreamName = randomIdentifier();
@@ -1123,7 +1188,6 @@ private static void assertTranslogOperation(
      * Assert that we can still search by synthetic _id after restoring index from snapshot
      */
     public void testCreateSnapshot() throws IOException {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
 
         // create index
@@ -1243,8 +1307,6 @@ public void testCreateSnapshot() throws IOException {
     }
 
     public void testMerge() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, 1, 0, rarely());
 
@@ -1423,8 +1485,10 @@ private static void putDataStreamTemplate(
         boolean useNestedDocs
     ) throws IOException {
         final var settings = indexSettings(primaries, replicas).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
-            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
-            .put(IndexSettings.SYNTHETIC_ID.getKey(), true);
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1);
+        if (randomBoolean()) {
+            settings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
+        }
         if (randomBoolean()) {
             settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC);
             settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), randomBoolean());

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.Before;
 
@@ -40,10 +39,6 @@
 
 public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
 
-    static {
-        clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-    }
-
     private static final String COMPONENT_TEMPLATE = """
         {
             "template": {

rest-api-spec/build.gradle

@@ -140,4 +140,16 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
     "get/100_synthetic_source/fields with ignore_malformed",
     "Malformed values are now stored in binary doc values which sort differently than stored fields"
   )
+  task.skipTest("delete/70_tsdb/basic tsdb delete", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/routing_path matches object", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/ids query", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/create operation on top of old document fails over bulk", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/delete over _bulk", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/delete", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/index a new document on top of an old one", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/generates a consistent id", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/create operation on top of old document fails", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/get", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/routing_path matches deep object", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/index a new document on top of an old one over bulk", "ids have changed after introduction of synthetic id")
 })

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -38,7 +38,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .feature(FeatureFlag.RANDOM_SAMPLING)
         .feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS)
-        .feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml

@@ -1,7 +1,7 @@
 ---
 "basic tsdb delete":
   - skip:
-      cluster_features: "index.time_series_synthetic_id"
+      cluster_features: "index.time_series_synthetic_id_default"
       reason: when cluster has synthetic_id feature use 71_tsdb_synthetic_id_false or 72_tsdb_synthetic_id_true instead
   - requires:
       cluster_features: ["gte_v8.13.0"]

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/71_tsdb_synthetic_id_false.yml

@@ -1,7 +1,7 @@
 ---
 "basic tsdb delete":
   - requires:
-      cluster_features: "index.time_series_synthetic_id"
+      cluster_features: "index.time_series_synthetic_id_default"
       reason: explicit synthetic_id false only applies when cluster has the feature
   - requires:
       cluster_features: ["gte_v8.13.0"]

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/72_tsdb_synthetic_id_true.yml

@@ -1,7 +1,7 @@
 ---
 "basic tsdb delete":
   - requires:
-      cluster_features: "index.time_series_synthetic_id"
+      cluster_features: "index.time_series_synthetic_id_default"
       reason: explicit synthetic_id true requires the feature
   - requires:
       cluster_features: ["gte_v8.13.0"]

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml

@@ -4,7 +4,7 @@ setup:
       reason: "tsdb index with nested field support enabled"
 
   - skip:
-      cluster_features: "index.time_series_synthetic_id"
+      cluster_features: "index.time_series_synthetic_id_default"
       reason: when cluster has synthetic_id feature use 161/162_nested_fields_synthetic_id_false/true instead
 
 ---

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/161_nested_fields_synthetic_id_false.yml

@@ -3,7 +3,7 @@ setup:
       cluster_features: ["mapper.tsdb_nested_field_support"]
       reason: "tsdb index with nested field support enabled"
   - requires:
-      cluster_features: ["index.time_series_synthetic_id"]
+      cluster_features: ["index.time_series_synthetic_id_default"]
       reason: "index.mapping.synthetic_id requires support for synthetic id in the cluster"
 
 ---

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/162_nested_fields_synthetic_id_true.yml

@@ -3,7 +3,7 @@ setup:
       cluster_features: ["mapper.tsdb_nested_field_support"]
       reason: "tsdb index with nested field support enabled"
   - requires:
-      cluster_features: ["index.time_series_synthetic_id"]
+      cluster_features: ["index.time_series_synthetic_id_default"]
       reason: "index.mapping.synthetic_id requires support for synthetic id in the cluster"
 
 ---

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml

@@ -5,7 +5,7 @@ setup:
       reason: _tsid hashing introduced in 8.13 and tsid routing changed in 8.14
 
   - skip:
-      cluster_features: "index.time_series_synthetic_id"
+      cluster_features: "index.time_series_synthetic_id_default"
       reason: when cluster has synthetic_id feature use 26_id_generation_synthetic_id_false instead
 
   - do: