diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index eca7caa961afe..7f28d7bfeffe6 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -137,6 +137,9 @@ public void testTimeRanges() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos")); if (randomBoolean()) { @@ -337,6 +340,9 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + settingsBuilder.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } request.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of("k8s*")) @@ -388,6 +394,9 @@ public void testSkippingShards() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var request = new TransportPutComposableIndexTemplateAction.Request("id1"); request.indexTemplate( ComposableIndexTemplate.builder() @@ -591,6 +600,9 @@ public void testReindexing() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() @@ -650,6 +662,9 @@ public void testAddDimensionToMapping() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName)) @@ -734,6 +749,9 @@ public void testDynamicStringDimensions() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() @@ -800,6 +818,9 @@ public void testDynamicDimensions() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index a9d15980df56c..bbd8c47df53a5 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -151,6 +151,9 @@ public void testIndexingGettingAndSearching() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var request = new TransportPutComposableIndexTemplateAction.Request("id"); request.indexTemplate( @@ -228,6 +231,9 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) { templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true); } + if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) { + templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true); + } var request = new TransportPutComposableIndexTemplateAction.Request("id"); request.indexTemplate( diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 5a0333ff88b66..862db950f2151 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -41,7 +41,7 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase { static { - clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG); + clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG).feature(FeatureFlag.TSDB_NO_SEQNO); } private static final String COMPONENT_TEMPLATE = """ diff --git a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java index f46a6d96f1a15..f073ea61727fc 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/IndexFeatures.java @@ -26,6 +26,8 @@ public Set getFeatures() { public static final NodeFeature TIME_SERIES_SYNTHETIC_ID = new NodeFeature("index.time_series_synthetic_id"); + public static final NodeFeature TIME_SERIES_NO_SEQNO = new NodeFeature("index.time_series_no_seqno"); + private static final NodeFeature SYNONYMS_SET_LENIENT_ON_NON_EXISTING = new NodeFeature("index.synonyms_set_lenient_on_non_existing"); private static final NodeFeature THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION = new NodeFeature( @@ -49,6 +51,7 @@ public Set getTestFeatures() { return Set.of( LOGSDB_NO_HOST_NAME_FIELD, TIME_SERIES_SYNTHETIC_ID, + TIME_SERIES_NO_SEQNO, SYNONYMS_SET_LENIENT_ON_NON_EXISTING, THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION, THROW_EXCEPTION_ON_INDEX_CREATION_IF_UNSUPPORTED_VALUE_TYPE_IN_ALIAS, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 639c30299f0fa..9c2f4925011b8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -963,7 +963,8 @@ private static int getIgnoreAboveDefaultValue(final Settings settings) { settings -> { final IndexMode indexMode = IndexSettings.MODE.get(settings); if ((indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) - && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS)) { + && (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS) + || IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).equals(IndexVersions.ZERO))) { return SeqNoFieldMapper.SeqNoIndexOptions.DOC_VALUES_ONLY.toString(); } else { return SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES.toString(); diff --git a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java index ccea7e3a4d7c2..f0d5e2735e224 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java @@ -372,6 +372,7 @@ private void runTimeSeriesForceMergeToOneSegment( final boolean pruneSequenceNumber, final boolean useSyntheticRecoverySource ) throws IOException { + assumeTrue("Synthetic id requires a feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG || useSyntheticId == false); assumeTrue( "Sequence number pruning requires a feature flag", IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG || pruneSequenceNumber == false @@ -578,6 +579,7 @@ public void testWrapForMergeUnwrapsSyntheticIdStoredFieldsReader() throws IOExce boolean syntheticRecoverySource = randomBoolean(); boolean pruneIdField = randomBoolean(); boolean pruneSequenceNumber = randomBoolean(); + assumeTrue("Synthetic id requires a feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG); assumeTrue( "Sequence number pruning requires a feature flag", IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG || (pruneSequenceNumber == false && pruneIdField == false) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index e194a8cfd236d..ad9191c2371e0 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -29,7 +29,8 @@ public enum FeatureFlag { TSDB_SYNTHETIC_ID_FEATURE_FLAG("es.tsdb_synthetic_id_feature_flag_enabled=true", Version.fromString("9.3.0"), null), ESQL_VIEWS("es.esql_views_feature_flag_enabled=true", Version.fromString("9.3.0"), null), EXTENDED_DOC_VALUES_PARAMS("es.extended_doc_values_options_feature_flag_enabled=true", Version.fromString("9.3.0"), null), - PROMETHEUS_FEATURE_FLAG("es.prometheus_feature_flag_enabled=true", Version.fromString("9.4.0"), null); + PROMETHEUS_FEATURE_FLAG("es.prometheus_feature_flag_enabled=true", Version.fromString("9.4.0"), null), + TSDB_NO_SEQNO("es.tsdb_no_tsbd_feature_flag_enabled=true", Version.fromString("9.4.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java index 932f44d69a66d..3e702df4cf540 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java @@ -23,7 +23,7 @@ public class TsdbIT extends AbstractLogsdbRollingUpgradeTestCase { - private static final String SYNTHETIC_ID_PLACEHOLDER = "$SYNTHETIC_ID_SETTING"; + private static final String EXTRA_INDEX_SETTINGS_PLACEHOLDER = "$SYNTHETIC_ID_SETTING"; // Do not access directly, use getTemplate(Boolean useSyntheticId) private static final String TEMPLATE = """ { @@ -83,15 +83,21 @@ public class TsdbIT extends AbstractLogsdbRollingUpgradeTestCase { } } } - """.formatted(SYNTHETIC_ID_PLACEHOLDER); + """.formatted(EXTRA_INDEX_SETTINGS_PLACEHOLDER); /** * Returns the template with optional synthetic_id setting. When {@code useSyntheticId} is null the setting is omitted; * when true/false, adds {@code "mapping": { "synthetic_id": true/false }} to index settings. */ - static String getTemplate(Boolean useSyntheticId) { - String replacement = useSyntheticId == null ? "" : ", \"mapping\": { \"synthetic_id\": " + useSyntheticId + " }"; - return TEMPLATE.replace(SYNTHETIC_ID_PLACEHOLDER, replacement); + static String getTemplate(Boolean useSyntheticId, Boolean disableSeqNo) { + StringBuilder replacement = new StringBuilder(); + if (useSyntheticId != null) { + replacement.append(", \"mapping\": { \"synthetic_id\": ").append(useSyntheticId).append(" }"); + } + if (disableSeqNo != null) { + replacement.append(", \"disable_sequence_numbers\": ").append(disableSeqNo); + } + return TEMPLATE.replace(EXTRA_INDEX_SETTINGS_PLACEHOLDER, replacement); } private static final String BULK = @@ -133,8 +139,8 @@ static String getTemplate(Boolean useSyntheticId) { """; public void testTsdbDataStream() throws Exception { - boolean hasSupport = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID); - Boolean useSyntheticId = hasSupport ? randomBoolean() : null; + Boolean useSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID) ? randomBoolean() : null; + Boolean disableSeqNo = oldClusterHasFeature(IndexFeatures.TIME_SERIES_NO_SEQNO) ? randomBoolean() : null; String dataStreamName = "k8s"; final String INDEX_TEMPLATE = """ @@ -148,7 +154,7 @@ public void testTsdbDataStream() throws Exception { String templateName = "1"; var putIndexTemplateRequest = new Request("POST", "/_index_template/" + templateName); putIndexTemplateRequest.setJsonEntity( - INDEX_TEMPLATE.replace("$TEMPLATE", getTemplate(useSyntheticId)).replace("$PATTERN", dataStreamName) + INDEX_TEMPLATE.replace("$TEMPLATE", getTemplate(useSyntheticId, disableSeqNo)).replace("$PATTERN", dataStreamName) ); assertOK(client().performRequest(putIndexTemplateRequest)); diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java index 469d4d3fc7fb5..17af0505967c8 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java @@ -44,13 +44,11 @@ public class TsdbIndexingRollingUpgradeIT extends AbstractLogsdbRollingUpgradeTe """; public void testIndexing() throws Exception { - // If cluster has support for synthetic id, randomly set index.mapping.synthetic_id to true/false - // If cluster doesn't have support, don't set index.mapping.synthetic_id at all (null), - // indicated by oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID)==false Boolean useSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID) ? randomBoolean() : null; + Boolean disableSeqNo = oldClusterHasFeature(IndexFeatures.TIME_SERIES_NO_SEQNO) ? randomBoolean() : null; String dataStreamName = "k9s"; - createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), getTemplate(useSyntheticId)); + createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), getTemplate(useSyntheticId, disableSeqNo)); Instant startTime = Instant.now().minusSeconds(60 * 60); bulkIndex(dataStreamName, 4, 1024, startTime, TsdbIndexingRollingUpgradeIT::docSupplier); @@ -65,6 +63,12 @@ public void testIndexing() throws Exception { if (useSyntheticId != null) { assertThat(((Map) settings.get("settings")).get(IndexSettings.SYNTHETIC_ID.getKey()), equalTo(useSyntheticId.toString())); } + if (disableSeqNo != null) { + assertThat( + ((Map) settings.get("settings")).get(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey()), + equalTo(disableSeqNo.toString()) + ); + } var mapping = getIndexMappingAsMap(firstBackingIndex); assertThat(