Synthetic id field for time_series indices (remove feature flag)

docs/changelog/144184.yaml

@@ -0,0 +1,17 @@
+area: TSDB
+pr: 144184
+summary: By using synthetic `_id` field for indices in time_series mode, storage footprint is significantly reduced
+highlight:
+  title: Synthetic IDs for indices in time-series mode
+  body: |-
+    The `_id` field has a significant storage footprint in metrics applications, as it requires both storing and indexing
+    unique document identifiers that are rarely used for direct lookups. To alleviate this, we are introducing synthetic
+    IDs for indices in time-series mode. Instead of indexing the `_id` field, a Bloom filter is used for fast, lightweight
+    duplicate detection at ingest time. Lookups and operations that previously relied on `_id` are delegated to other
+    indexed fields on the document, such as timestamps, or dimension fields, preserving the same query and
+    retrieval functionality.
+
+    This offers up to 40% storage improvement for OTLP metrics and reduces the cpu overhead for segment merging due to the
+    lack of an inverted index for `_id` fields.
+  notable: true
+type: "enhancement"

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -134,9 +134,7 @@ public void testTimeRanges() throws Exception {
         if (randomBoolean()) {
             templateSettings.put("index.routing_path", "metricset");
         }
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }
@@ -343,9 +341,7 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception {
         if (randomBoolean()) {
             settingsBuilder.put("index.routing_path", "metricset");
         }
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             settingsBuilder.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }
@@ -397,9 +393,7 @@ public void testSkippingShards() throws Exception {
         var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos"));
         {
             var templateSettings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset");
-            if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-                templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-            }
+            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
             if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
                 templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
             }
@@ -469,12 +463,10 @@ public void testTrimId() throws Exception {
             .put("index.number_of_replicas", 0)
             // Reduce sync interval to speedup this integraton test,
             // otherwise by default it will take 30 seconds before minimum retained seqno is updated:
-            .put("index.soft_deletes.retention_lease.sync_interval", "100ms");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) {
+            .put("index.soft_deletes.retention_lease.sync_interval", "100ms")
             // This test checks that _id's are pruned, that only applies
             // when regular _id's are used.
-            indexSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), false);
-        }
+            .put(IndexSettings.SYNTHETIC_ID.getKey(), false);
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG) {
             // This test relies on sequence numbers to verify ids.
             indexSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false);
@@ -607,9 +599,7 @@ public void testReindexing() throws Exception {
         String dataStreamName = "my-ds";
         String reindexedDataStreamName = "my-reindexed-ds";
         var templateSettings = Settings.builder().put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }
@@ -668,10 +658,8 @@ public void testAddDimensionToMapping() throws Exception {
         boolean indexDimensionsTsidStrategyEnabled = randomBoolean();
         var templateSettings = Settings.builder()
             .put("index.mode", "time_series")
-            .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabled);
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+            .put("index.dimensions_tsid_strategy_enabled", indexDimensionsTsidStrategyEnabled)
+            .put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }
@@ -756,9 +744,7 @@ public void testAddDimensionToMapping() throws Exception {
     public void testDynamicStringDimensions() throws Exception {
         String dataStreamName = "my-ds";
         var templateSettings = Settings.builder().put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }
@@ -825,9 +811,7 @@ public void testDynamicStringDimensions() throws Exception {
     public void testDynamicDimensions() throws Exception {
         String dataStreamName = "my-ds";
         var templateSettings = Settings.builder().put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

@@ -149,9 +149,7 @@ protected Settings nodeSettings() {
 
     public void testIndexingGettingAndSearching() throws Exception {
         var templateSettings = indexSettings(randomIntBetween(2, 10), 0).put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }
@@ -235,9 +233,7 @@ public void testIndexingGettingAndSearching() throws Exception {
     public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {
         String dataStreamName = "k8s";
         var templateSettings = indexSettings(8, 0).put("index.mode", "time_series");
-        if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
-            templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
-        }
+        templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), randomBoolean());
         if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
             templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
         }

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java

@@ -151,7 +151,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testInvalidIndexMode() {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final var indexName = randomIdentifier();
         var randomNonTsdbIndexMode = randomValueOtherThan(IndexMode.TIME_SERIES, () -> randomFrom(IndexMode.values()));
 
@@ -177,7 +176,6 @@ public void testInvalidIndexMode() {
     }
 
     public void testInvalidCodec() {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final var indexName = randomIdentifier();
         internalCluster().startDataOnlyNode();
         var randomNonDefaultCodec = randomFrom(
@@ -210,7 +208,6 @@ public void testInvalidCodec() {
     }
 
     public void testSyntheticId() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, randomIntBetween(1, 5), 0, useNestedDocs);
@@ -456,7 +453,6 @@ enum Operation {
     }
 
     public void testGetFromTranslogBySyntheticId() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, 1, 0, useNestedDocs);
@@ -583,7 +579,6 @@ public void testGetFromTranslogBySyntheticId() throws Exception {
     }
 
     public void testRecoveredOperations() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
 
         // ensure a couple of nodes to have some operations coordinated
@@ -828,7 +823,6 @@ enum Operation {
     }
 
     public void testRecoverOperationsFromLocalTranslog() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
 
         final var dataStreamName = randomIdentifier();
@@ -1136,7 +1130,6 @@ private static void assertTranslogOperation(
      * Assert that we can still search by synthetic _id after restoring index from snapshot
      */
     public void testCreateSnapshot() throws IOException {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         final boolean useNestedDocs = rarely();
 
         // create index
@@ -1256,8 +1249,6 @@ public void testCreateSnapshot() throws IOException {
     }
 
     public void testMerge() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, 1, 0, rarely());
 
@@ -1306,8 +1297,6 @@ public void testMerge() throws Exception {
     }
 
     public void testBestCompressionCodec() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-
         String indexName = randomIndexName();
 
         // Set best_compression codec
@@ -1376,8 +1365,6 @@ public void testBestCompressionCodec() throws Exception {
     }
 
     public void testDefaultSetting() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-
         String indexName = randomIndexName();
 
         // Don't set IndexSettings.SYNTHETIC_ID to test default behavior.
@@ -1427,7 +1414,7 @@ public void testDefaultSetting() throws Exception {
         GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(TEST_REQUEST_TIMEOUT, indexName).get();
         String versionSetting = getSettingsResponse.getSetting(indexName, IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey());
         IndexVersion version = IndexVersion.fromId(Integer.parseInt(versionSetting));
-        assertTrue(version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT));
+        assertTrue(version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD));
         String syntheticIdSetting = getSettingsResponse.getSetting(indexName, IndexSettings.SYNTHETIC_ID.getKey());
         assertThat(syntheticIdSetting, Matchers.nullValue());
 
@@ -1442,8 +1429,6 @@ public void testDefaultSetting() throws Exception {
     }
 
     public void testBloomFilterSettings() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-
         final var dataStreamName = randomIdentifier();
         final var maxSize = ByteSizeValue.ofKb(64);
         // small_segment_max_docs must be strictly less than large_segment_min_docs; high >= low bits/doc
@@ -1597,7 +1582,6 @@ public void testBloomFilterSettings() throws Exception {
      * synthetic id until node B has been upgraded.
      */
     public void testIndexCreationIsBlockByIndexVersion() {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         String indexName = randomIndexName();
         // IndexVersion is too low for synthetic id to be allowed
         IndexVersion tooLowIndexVersion = IndexVersionUtils.randomPreviousCompatibleWriteVersion(
@@ -1788,8 +1772,10 @@ private static void putDataStreamTemplate(
     ) throws IOException {
         final var settings = indexSettings(primaries, replicas).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
-            .put(IndexSettings.SYNTHETIC_ID.getKey(), true)
-            .put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false);  // Sequence numbers are needed for id validation.
+            .put(IndexSettings.SYNTHETIC_ID.getKey(), true);
+        if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG) {
+            settings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false); // Sequence numbers are needed for id validation.
+        }
         if (randomBoolean()) {
             settings.put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomValidCodec());
         }
@@ -1885,7 +1871,6 @@ private static IndexDiskUsageStats diskUsage(String indexName) {
      * {@link IndexShard#newChangesSnapshot}, and that GET/search by synthetic _id work correctly after each scenario.
      */
     public void testNoopTombstones() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         internalCluster().startMasterOnlyNode();
         List<String> dataNodeNames = internalCluster().startDataOnlyNodes(2);
 
@@ -2306,8 +2291,6 @@ private void assertGetAndSearchById(Map<String, String> docsIndicesById, String
     }
 
     public void testTermInSetQueryWithSyntheticIds() throws Exception {
-        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
-
         final var dataStreamName = randomIdentifier();
         putDataStreamTemplate(dataStreamName, 1, 0, rarely());
 

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java

@@ -41,7 +41,7 @@
 public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
 
     static {
-        clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG).feature(FeatureFlag.TSDB_NO_SEQNO);
+        clusterConfig = config -> config.feature(FeatureFlag.TSDB_NO_SEQNO);
     }
 
     private static final String COMPONENT_TEMPLATE = """

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -18,6 +18,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
@@ -322,6 +324,15 @@ private void createTsdbIndex() throws IOException {
         indexSpec.array("routing_path", new String[] { "dim" });
         indexSpec.field("time_series.start_time", 1L);
         indexSpec.field("time_series.end_time", DateUtils.MAX_MILLIS_BEFORE_9999 - 1);
+
+        // Disable synthetic id if old cluster is in the version range that enables
+        // synthetic id behind feature flag. Otherwise, the index will change behavior
+        // once upgrade is finished and cause the test to fail.
+        IndexVersion oldClusterIndexVersion = getOldClusterIndexVersion();
+        if (oldClusterIndexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT)
+            && oldClusterIndexVersion.before(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD)) {
+            indexSpec.field("mapping.synthetic_id", "false");
+        }
         indexSpec.endObject().endObject();
         createIndex.setJsonEntity(Strings.toString(indexSpec.endObject()));
         client().performRequest(createIndex);

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -38,7 +38,6 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .feature(FeatureFlag.RANDOM_SAMPLING)
         .feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS)
-        .feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -38,7 +38,6 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .feature(FeatureFlag.RANDOM_SAMPLING)
         .feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS)
-        .feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -2637,11 +2637,10 @@ IndexMetadata build(boolean repair) {
 
         private static boolean shouldUseTimeSeriesSyntheticId(boolean isTsdb, IndexVersion version, Settings settings) {
             String codecSetting = settings.get(EngineConfig.INDEX_CODEC_SETTING.getKey());
-            if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG
-                && isTsdb
+            if (isTsdb
                 && version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)
                 && (codecSetting == null || IndexSettings.isValidCodecForSyntheticId(codecSetting, version))) {
-                boolean defaultValue = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT);
+                boolean defaultValue = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT_PROD);
                 return settings.getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), defaultValue);
             }
             return false;