Skip to content

[TEST] Add coverage for disabling sequence numbers in TSDB ITs #143897

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kkrik-es merged 19 commits into from
Mar 11, 2026
Merged

[TEST] Add coverage for disabling sequence numbers in TSDB ITs #143897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
0eacbe7
Disable sequence numbers in TSDB integration tests
kkrik-es Mar 9, 2026
cdbcbb6
[CI] Auto commit changes from spotless
Mar 9, 2026
63536ec
fix compilation test
kkrik-es Mar 9, 2026
d35b289
Merge remote-tracking branch 'origin/tsdb/seqno-trim-tests' into tsdb…
kkrik-es Mar 9, 2026
a5e0079
fix test
kkrik-es Mar 9, 2026
efcab16
Merge branch 'main' into tsdb/seqno-trim-tests
kkrik-es Mar 9, 2026
c4416c8
Merge branch 'main' into tsdb/seqno-trim-tests
kkrik-es Mar 10, 2026
1036968
fix rest tests
kkrik-es Mar 10, 2026
e3c12f3
Merge remote-tracking branch 'origin/tsdb/seqno-trim-tests' into tsdb…
kkrik-es Mar 10, 2026
5a784bb
fix rest tests
kkrik-es Mar 10, 2026
55d49d4
Merge branch 'main' into tsdb/seqno-trim-tests
kkrik-es Mar 10, 2026
577ae6a
fix more tests
kkrik-es Mar 10, 2026
e3812ae
fix TSDBSyntheticIdsIT
kkrik-es Mar 10, 2026
b936cfc
[CI] Auto commit changes from spotless
Mar 10, 2026
d98849c
Merge branch 'main' into tsdb/seqno-trim-tests
kkrik-es Mar 11, 2026
2a6cfaa
revert changes in TSDBSyntheticIdsIT
kkrik-es Mar 11, 2026
86fb0f2
fix
kkrik-es Mar 11, 2026
0d8d826
Merge branch 'main' into tsdb/seqno-trim-tests
kkrik-es Mar 11, 2026
294965c
Merge branch 'main' into tsdb/seqno-trim-tests
kkrik-es Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ...ta-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ public void testTimeRanges() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos"));

if (randomBoolean()) {
Expand Down Expand Up @@ -337,6 +340,9 @@ public void testTsdbTemplatesNoKeywordFieldType() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
settingsBuilder.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
settingsBuilder.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
request.indexTemplate(
ComposableIndexTemplate.builder()
.indexPatterns(List.of("k8s*"))
Expand Down Expand Up @@ -388,6 +394,9 @@ public void testSkippingShards() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
var request = new TransportPutComposableIndexTemplateAction.Request("id1");
request.indexTemplate(
ComposableIndexTemplate.builder()
Expand Down Expand Up @@ -591,6 +600,9 @@ public void testReindexing() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
putTemplateRequest.indexTemplate(
ComposableIndexTemplate.builder()
Expand Down Expand Up @@ -650,6 +662,9 @@ public void testAddDimensionToMapping() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
putTemplateRequest.indexTemplate(
ComposableIndexTemplate.builder()
.indexPatterns(List.of(dataStreamName))
Expand Down Expand Up @@ -734,6 +749,9 @@ public void testDynamicStringDimensions() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
putTemplateRequest.indexTemplate(
ComposableIndexTemplate.builder()
Expand Down Expand Up @@ -800,6 +818,9 @@ public void testDynamicDimensions() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}
var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id");
putTemplateRequest.indexTemplate(
ComposableIndexTemplate.builder()
Expand Down
6 changes: 6 additions & 0 deletions ...src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,9 @@ public void testIndexingGettingAndSearching() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}

var request = new TransportPutComposableIndexTemplateAction.Request("id");
request.indexTemplate(
Expand Down Expand Up @@ -228,6 +231,9 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {
if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.SYNTHETIC_ID.getKey(), true);
}
if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG && randomBoolean()) {
templateSettings.put(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), true);
}

var request = new TransportPutComposableIndexTemplateAction.Request("id");
request.indexTemplate(
Expand Down
2 changes: 1 addition & 1 deletion ...ata-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {

static {
clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG).feature(FeatureFlag.TSDB_NO_SEQNO);
}

private static final String COMPONENT_TEMPLATE = """
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/java/org/elasticsearch/index/IndexFeatures.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ public Set<NodeFeature> getFeatures() {

public static final NodeFeature TIME_SERIES_SYNTHETIC_ID = new NodeFeature("index.time_series_synthetic_id");

public static final NodeFeature TIME_SERIES_NO_SEQNO = new NodeFeature("index.time_series_no_seqno");

private static final NodeFeature SYNONYMS_SET_LENIENT_ON_NON_EXISTING = new NodeFeature("index.synonyms_set_lenient_on_non_existing");

private static final NodeFeature THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION = new NodeFeature(
Expand All @@ -49,6 +51,7 @@ public Set<NodeFeature> getTestFeatures() {
return Set.of(
LOGSDB_NO_HOST_NAME_FIELD,
TIME_SERIES_SYNTHETIC_ID,
TIME_SERIES_NO_SEQNO,
SYNONYMS_SET_LENIENT_ON_NON_EXISTING,
THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION,
THROW_EXCEPTION_ON_INDEX_CREATION_IF_UNSUPPORTED_VALUE_TYPE_IN_ALIAS,
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/elasticsearch/index/IndexSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -963,7 +963,8 @@ private static int getIgnoreAboveDefaultValue(final Settings settings) {
settings -> {
final IndexMode indexMode = IndexSettings.MODE.get(settings);
if ((indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES)
&& IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS)) {
&& (IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS)
|| IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).equals(IndexVersions.ZERO))) {
Copy link
Copy Markdown
Member Author

@kkrik-es kkrik-es Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

validateIndexTemplateV2 does index settings validation without necessarily populating the index version. We can either fix that, or bypass the check when the version's missing as above.

return SeqNoFieldMapper.SeqNoIndexOptions.DOC_VALUES_ONLY.toString();
} else {
return SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES.toString();
Expand Down
2 changes: 2 additions & 0 deletions server/src/test/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicyTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ private void runTimeSeriesForceMergeToOneSegment(
final boolean pruneSequenceNumber,
final boolean useSyntheticRecoverySource
) throws IOException {
assumeTrue("Synthetic id requires a feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG || useSyntheticId == false);
assumeTrue(
"Sequence number pruning requires a feature flag",
IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG || pruneSequenceNumber == false
Expand Down Expand Up @@ -578,6 +579,7 @@ public void testWrapForMergeUnwrapsSyntheticIdStoredFieldsReader() throws IOExce
boolean syntheticRecoverySource = randomBoolean();
boolean pruneIdField = randomBoolean();
boolean pruneSequenceNumber = randomBoolean();
assumeTrue("Synthetic id requires a feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
assumeTrue(
"Sequence number pruning requires a feature flag",
IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG || (pruneSequenceNumber == false && pruneIdField == false)
Expand Down
3 changes: 2 additions & 1 deletion test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ public enum FeatureFlag {
TSDB_SYNTHETIC_ID_FEATURE_FLAG("es.tsdb_synthetic_id_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
ESQL_VIEWS("es.esql_views_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
EXTENDED_DOC_VALUES_PARAMS("es.extended_doc_values_options_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
PROMETHEUS_FEATURE_FLAG("es.prometheus_feature_flag_enabled=true", Version.fromString("9.4.0"), null);
PROMETHEUS_FEATURE_FLAG("es.prometheus_feature_flag_enabled=true", Version.fromString("9.4.0"), null),
TSDB_NO_SEQNO("es.tsdb_no_tsbd_feature_flag_enabled=true", Version.fromString("9.4.0"), null);

public final String systemProperty;
public final Version from;
Expand Down
22 changes: 14 additions & 8 deletions ...ogsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

public class TsdbIT extends AbstractLogsdbRollingUpgradeTestCase {

private static final String SYNTHETIC_ID_PLACEHOLDER = "$SYNTHETIC_ID_SETTING";
private static final String EXTRA_INDEX_SETTINGS_PLACEHOLDER = "$SYNTHETIC_ID_SETTING";
// Do not access directly, use getTemplate(Boolean useSyntheticId)
private static final String TEMPLATE = """
{
Expand Down Expand Up @@ -83,15 +83,21 @@ public class TsdbIT extends AbstractLogsdbRollingUpgradeTestCase {
}
}
}
""".formatted(SYNTHETIC_ID_PLACEHOLDER);
""".formatted(EXTRA_INDEX_SETTINGS_PLACEHOLDER);

/**
* Returns the template with optional synthetic_id setting. When {@code useSyntheticId} is null the setting is omitted;
* when true/false, adds {@code "mapping": { "synthetic_id": true/false }} to index settings.
*/
static String getTemplate(Boolean useSyntheticId) {
String replacement = useSyntheticId == null ? "" : ", \"mapping\": { \"synthetic_id\": " + useSyntheticId + " }";
return TEMPLATE.replace(SYNTHETIC_ID_PLACEHOLDER, replacement);
static String getTemplate(Boolean useSyntheticId, Boolean disableSeqNo) {
StringBuilder replacement = new StringBuilder();
if (useSyntheticId != null) {
replacement.append(", \"mapping\": { \"synthetic_id\": ").append(useSyntheticId).append(" }");
}
if (disableSeqNo != null) {
replacement.append(", \"disable_sequence_numbers\": ").append(disableSeqNo);
}
return TEMPLATE.replace(EXTRA_INDEX_SETTINGS_PLACEHOLDER, replacement);
}

private static final String BULK =
Expand Down Expand Up @@ -133,8 +139,8 @@ static String getTemplate(Boolean useSyntheticId) {
""";

public void testTsdbDataStream() throws Exception {
boolean hasSupport = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID);
Boolean useSyntheticId = hasSupport ? randomBoolean() : null;
Boolean useSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID) ? randomBoolean() : null;
Boolean disableSeqNo = oldClusterHasFeature(IndexFeatures.TIME_SERIES_NO_SEQNO) ? randomBoolean() : null;

String dataStreamName = "k8s";
final String INDEX_TEMPLATE = """
Expand All @@ -148,7 +154,7 @@ public void testTsdbDataStream() throws Exception {
String templateName = "1";
var putIndexTemplateRequest = new Request("POST", "/_index_template/" + templateName);
putIndexTemplateRequest.setJsonEntity(
INDEX_TEMPLATE.replace("$TEMPLATE", getTemplate(useSyntheticId)).replace("$PATTERN", dataStreamName)
INDEX_TEMPLATE.replace("$TEMPLATE", getTemplate(useSyntheticId, disableSeqNo)).replace("$PATTERN", dataStreamName)
);
assertOK(client().performRequest(putIndexTemplateRequest));

Expand Down
12 changes: 8 additions & 4 deletions ...de/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/TsdbIndexingRollingUpgradeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,11 @@ public class TsdbIndexingRollingUpgradeIT extends AbstractLogsdbRollingUpgradeTe
""";

public void testIndexing() throws Exception {
// If cluster has support for synthetic id, randomly set index.mapping.synthetic_id to true/false
// If cluster doesn't have support, don't set index.mapping.synthetic_id at all (null),
// indicated by oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID)==false
Boolean useSyntheticId = oldClusterHasFeature(IndexFeatures.TIME_SERIES_SYNTHETIC_ID) ? randomBoolean() : null;
Boolean disableSeqNo = oldClusterHasFeature(IndexFeatures.TIME_SERIES_NO_SEQNO) ? randomBoolean() : null;

String dataStreamName = "k9s";
createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), getTemplate(useSyntheticId));
createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), getTemplate(useSyntheticId, disableSeqNo));

Instant startTime = Instant.now().minusSeconds(60 * 60);
bulkIndex(dataStreamName, 4, 1024, startTime, TsdbIndexingRollingUpgradeIT::docSupplier);
Expand All @@ -65,6 +63,12 @@ public void testIndexing() throws Exception {
if (useSyntheticId != null) {
assertThat(((Map<?, ?>) settings.get("settings")).get(IndexSettings.SYNTHETIC_ID.getKey()), equalTo(useSyntheticId.toString()));
}
if (disableSeqNo != null) {
assertThat(
((Map<?, ?>) settings.get("settings")).get(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey()),
equalTo(disableSeqNo.toString())
);
}

var mapping = getIndexMappingAsMap(firstBackingIndex);
assertThat(
Expand Down
Loading