diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 5ebdbd272f3fe..7252d31d838c5 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.datastreams.lifecycle.ErrorEntry; @@ -41,12 +42,16 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction; +import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction; import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthIndicatorService; import org.elasticsearch.health.Diagnosis; import org.elasticsearch.health.GetHealthAction; @@ -58,14 +63,20 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.indices.ExecutorNames; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.junit.After; import java.io.IOException; +import java.time.Clock; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -73,6 +84,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; @@ -82,6 +94,8 @@ import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService.DATA_STREAM_MERGE_POLICY_TARGET_FLOOR_SEGMENT_SETTING; import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService.ONE_HUNDRED_MB; import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService.TARGET_MERGE_FACTOR_VALUE; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleServiceIT.TestSystemDataStreamPlugin.SYSTEM_DATA_STREAM_NAME; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleServiceIT.TestSystemDataStreamPlugin.SYSTEM_DATA_STREAM_RETENTION_DAYS; import static org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthIndicatorService.STAGNATING_BACKING_INDICES_DIAGNOSIS_DEF; import static org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthIndicatorService.STAGNATING_INDEX_IMPACT; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; @@ -102,7 +116,7 @@ public class DataStreamLifecycleServiceIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class); + return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class, TestSystemDataStreamPlugin.class); } @Override @@ -173,6 +187,116 @@ public void testRolloverAndRetention() throws Exception { }); } + @SuppressWarnings("unchecked") + public void testSystemDataStreamRetention() throws Exception { + /* + * This test makes sure that global data stream retention is ignored by system data streams, and that the configured retention + * for a system data stream is respected instead. + */ + Iterable dataStreamLifecycleServices = internalCluster().getInstances(DataStreamLifecycleService.class); + Clock clock = Clock.systemUTC(); + AtomicLong now = new AtomicLong(clock.millis()); + dataStreamLifecycleServices.forEach(dataStreamLifecycleService -> dataStreamLifecycleService.setNowSupplier(now::get)); + try { + // Putting in place a global retention that we expect will be ignored by the system data stream: + final int globalRetentionSeconds = 10; + client().execute( + PutDataStreamGlobalRetentionAction.INSTANCE, + new PutDataStreamGlobalRetentionAction.Request( + TimeValue.timeValueSeconds(globalRetentionSeconds), + TimeValue.timeValueSeconds(globalRetentionSeconds) + ) + ).actionGet(); + try { + + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet(); + indexDocs(SYSTEM_DATA_STREAM_NAME, 1); + /* + * First we advance the time to well beyond the global retention (10s) but well under the configured retention (100d). + * We expect to see that rollover has occurred but that the old index has not been deleted since the global retention is + * ignored. + */ + now.addAndGet(TimeValue.timeValueSeconds(3 * globalRetentionSeconds).millis()); + assertBusy(() -> { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( + new String[] { SYSTEM_DATA_STREAM_NAME } + ); + GetDataStreamAction.Response getDataStreamResponse = client().execute( + GetDataStreamAction.INSTANCE, + getDataStreamRequest + ).actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(SYSTEM_DATA_STREAM_NAME)); + List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); + assertThat(backingIndices.size(), equalTo(2)); // global retention is ignored + // we expect the data stream to have two backing indices since the effective retention is 100 days + String writeIndex = backingIndices.get(1).getName(); + assertThat(writeIndex, backingIndexEqualTo(SYSTEM_DATA_STREAM_NAME, 2)); + }); + + // Now we advance the time to well beyond the configured retention. We expect that the older index will have been deleted. + now.addAndGet(TimeValue.timeValueDays(3 * TestSystemDataStreamPlugin.SYSTEM_DATA_STREAM_RETENTION_DAYS).millis()); + assertBusy(() -> { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( + new String[] { SYSTEM_DATA_STREAM_NAME } + ); + GetDataStreamAction.Response getDataStreamResponse = client().execute( + GetDataStreamAction.INSTANCE, + getDataStreamRequest + ).actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(SYSTEM_DATA_STREAM_NAME)); + List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); + assertThat(backingIndices.size(), equalTo(1)); // global retention is ignored + // we expect the data stream to have only one backing index, the write one, with generation 2 + // as generation 1 would've been deleted by the data stream lifecycle given the configuration + String writeIndex = backingIndices.get(0).getName(); + assertThat(writeIndex, backingIndexEqualTo(SYSTEM_DATA_STREAM_NAME, 2)); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + ToXContent.Params withEffectiveRetention = new ToXContent.MapParams( + DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS + ); + getDataStreamResponse.getDataStreams() + .get(0) + .toXContent( + builder, + withEffectiveRetention, + getDataStreamResponse.getRolloverConfiguration(), + getDataStreamResponse.getGlobalRetention() + ); + String serialized = Strings.toString(builder); + Map resultMap = XContentHelper.convertToMap( + XContentType.JSON.xContent(), + serialized, + randomBoolean() + ); + assertNotNull(resultMap); + Map lifecycleMap = (Map) resultMap.get("lifecycle"); + assertNotNull(lifecycleMap); + assertThat( + lifecycleMap.get("data_retention"), + equalTo(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS).getStringRep()) + ); + assertThat( + lifecycleMap.get("effective_retention"), + equalTo(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS).getStringRep()) + ); + assertThat(lifecycleMap.get("retention_determined_by"), equalTo("data_stream_configuration")); + assertThat(lifecycleMap.get("enabled"), equalTo(true)); + } + }); + + client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME)).actionGet(); + } finally { + client().execute(DeleteDataStreamGlobalRetentionAction.INSTANCE, new DeleteDataStreamGlobalRetentionAction.Request()); + } + } finally { + dataStreamLifecycleServices.forEach(dataStreamLifecycleService -> dataStreamLifecycleService.setNowSupplier(clock::millis)); + } + } + public void testOriginationDate() throws Exception { /* * In this test, we set up a datastream with 7 day retention. Then we add two indices to it -- one with an origination date 365 @@ -880,4 +1004,51 @@ static void updateLifecycle(String dataStreamName, TimeValue dataRetention) { ); assertAcked(client().execute(PutDataStreamLifecycleAction.INSTANCE, putDataLifecycleRequest)); } + + /* + * This test plugin adds `.system-test` as a known system data stream. The data stream is not created by this plugin. But if it is + * created, it will be a system data stream. + */ + public static class TestSystemDataStreamPlugin extends Plugin implements SystemIndexPlugin { + public static final String SYSTEM_DATA_STREAM_NAME = ".system-test"; + public static final int SYSTEM_DATA_STREAM_RETENTION_DAYS = 100; + + @Override + public String getFeatureName() { + return "test"; + } + + @Override + public String getFeatureDescription() { + return "test"; + } + + @Override + public Collection getSystemDataStreamDescriptors() { + return List.of( + new SystemDataStreamDescriptor( + SYSTEM_DATA_STREAM_NAME, + "test", + SystemDataStreamDescriptor.Type.INTERNAL, + ComposableIndexTemplate.builder() + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .indexPatterns(List.of(DataStream.BACKING_INDEX_PREFIX + SYSTEM_DATA_STREAM_NAME + "*")) + .template( + new Template( + Settings.EMPTY, + null, + null, + DataStreamLifecycle.newBuilder() + .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) + .build() + ) + ) + .build(), + Map.of(), + List.of(), + ExecutorNames.DEFAULT_SYSTEM_INDEX_THREAD_POOLS + ) + ); + } + } } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java index 7120196176928..2723637b2959b 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java @@ -28,7 +28,10 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction; +import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.plugins.Plugin; @@ -46,6 +49,8 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleServiceIT.TestSystemDataStreamPlugin.SYSTEM_DATA_STREAM_NAME; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleServiceIT.TestSystemDataStreamPlugin.SYSTEM_DATA_STREAM_RETENTION_DAYS; import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -59,7 +64,11 @@ public class ExplainDataStreamLifecycleIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class); + return List.of( + DataStreamsPlugin.class, + MockTransportService.TestPlugin.class, + DataStreamLifecycleServiceIT.TestSystemDataStreamPlugin.class + ); } @Override @@ -194,6 +203,67 @@ public void testExplainLifecycle() throws Exception { } } + public void testSystemExplainLifecycle() throws Exception { + /* + * This test makes sure that for system data streams, we correctly ignore the global retention when calling + * ExplainDataStreamLifecycle. It is very similar to testExplainLifecycle, but only focuses on the retention for a system index. + */ + // Putting in place a global retention that we expect will be ignored by the system data stream: + final int globalRetentionSeconds = 10; + client().execute( + PutDataStreamGlobalRetentionAction.INSTANCE, + new PutDataStreamGlobalRetentionAction.Request( + TimeValue.timeValueSeconds(globalRetentionSeconds), + TimeValue.timeValueSeconds(globalRetentionSeconds) + ) + ).actionGet(); + try { + String dataStreamName = SYSTEM_DATA_STREAM_NAME; + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); + + indexDocs(dataStreamName, 1); + + assertBusy(() -> { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName }); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); + List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); + assertThat(backingIndices.size(), equalTo(2)); + String backingIndex = backingIndices.get(0).getName(); + assertThat(backingIndex, backingIndexEqualTo(dataStreamName, 1)); + String writeIndex = backingIndices.get(1).getName(); + assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); + }); + + ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request( + new String[] { + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + DataStream.getDefaultBackingIndexName(dataStreamName, 2) } + ); + ExplainDataStreamLifecycleAction.Response response = client().execute( + ExplainDataStreamLifecycleAction.INSTANCE, + explainIndicesRequest + ).actionGet(); + assertThat(response.getIndices().size(), is(2)); + // we requested the explain for indices with the default include_details=false + assertThat(response.getRolloverConfiguration(), nullValue()); + for (ExplainIndexDataStreamLifecycle explainIndex : response.getIndices()) { + assertThat(explainIndex.isManagedByLifecycle(), is(true)); + assertThat(explainIndex.getIndexCreationDate(), notNullValue()); + assertThat(explainIndex.getLifecycle(), notNullValue()); + assertThat( + explainIndex.getLifecycle().getDataStreamRetention(), + equalTo(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) + ); + } + } finally { + client().execute(DeleteDataStreamGlobalRetentionAction.INSTANCE, new DeleteDataStreamGlobalRetentionAction.Request()); + } + } + public void testExplainLifecycleForIndicesWithErrors() throws Exception { // empty lifecycle contains the default rollover DataStreamLifecycle lifecycle = new DataStreamLifecycle(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 9e3dd5cc1a3ba..27beb92e23180 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -787,7 +787,8 @@ private Set maybeExecuteRollover(ClusterState state, DataStream dataStrea RolloverRequest rolloverRequest = getDefaultRolloverRequest( rolloverConfiguration, dataStream.getName(), - dataStream.getLifecycle().getEffectiveDataRetention(globalRetentionResolver.resolve(state)) + dataStream.getLifecycle() + .getEffectiveDataRetention(dataStream.isSystem() ? null : globalRetentionResolver.resolve(state)) ); transportActionsDeduplicator.executeOnce( rolloverRequest, @@ -840,7 +841,8 @@ private Set maybeExecuteRetention(ClusterState state, DataStream dataStre Set indicesToBeRemoved = new HashSet<>(); // We know that there is lifecycle and retention because there are indices to be deleted assert dataStream.getLifecycle() != null; - TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention); + TimeValue effectiveDataRetention = dataStream.getLifecycle() + .getEffectiveDataRetention(dataStream.isSystem() ? null : globalRetention); for (Index index : backingIndicesOlderThanRetention) { if (indicesToExcludeForRemainingRun.contains(index) == false) { IndexMetadata backingIndex = metadata.index(index); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java index 4c54189ee0111..a18095c555f12 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java @@ -105,8 +105,10 @@ public List determin List affectedDataStreams = new ArrayList<>(); for (DataStream dataStream : clusterState.metadata().dataStreams().values()) { if (dataStream.getLifecycle() != null) { - TimeValue previousEffectiveRetention = dataStream.getLifecycle().getEffectiveDataRetention(previousGlobalRetention); - TimeValue newEffectiveRetention = dataStream.getLifecycle().getEffectiveDataRetention(newGlobalRetention); + TimeValue previousEffectiveRetention = dataStream.getLifecycle() + .getEffectiveDataRetention(dataStream.isSystem() ? null : previousGlobalRetention); + TimeValue newEffectiveRetention = dataStream.getLifecycle() + .getEffectiveDataRetention(dataStream.isSystem() ? null : newGlobalRetention); if (Objects.equals(previousEffectiveRetention, newEffectiveRetention) == false) { affectedDataStreams.add( new UpdateDataStreamGlobalRetentionResponse.AffectedDataStream( diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java index ac5f46edb5ccc..fe5b3a1a378ff 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java @@ -94,7 +94,7 @@ protected void masterOperation( DataStream parentDataStream = indexAbstraction.getParentDataStream(); if (parentDataStream == null || parentDataStream.isIndexManagedByDataStreamLifecycle(idxMetadata.getIndex(), metadata::index) == false) { - explainIndices.add(new ExplainIndexDataStreamLifecycle(index, false, null, null, null, null, null)); + explainIndices.add(new ExplainIndexDataStreamLifecycle(index, false, false, null, null, null, null, null)); continue; } @@ -103,6 +103,7 @@ protected void masterOperation( ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( index, true, + parentDataStream.isSystem(), idxMetadata.getCreationDate(), rolloverInfo == null ? null : rolloverInfo.getTime(), generationDate, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java index deff083579800..7ac9eaae41a50 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java @@ -89,7 +89,8 @@ protected void masterOperation( .map( dataStream -> new GetDataStreamLifecycleAction.Response.DataStreamLifecycle( dataStream.getName(), - dataStream.getLifecycle() + dataStream.getLifecycle(), + dataStream.isSystem() ) ) .sorted(Comparator.comparing(GetDataStreamLifecycleAction.Response.DataStreamLifecycle::dataStreamName)) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java index 41e3e3a28ed5a..b9dc6d349873c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java @@ -15,10 +15,10 @@ import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.test.ClusterServiceUtils; @@ -31,6 +31,7 @@ import org.junit.BeforeClass; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.hamcrest.CoreMatchers.equalTo; @@ -129,7 +130,7 @@ public void testUpdateClusterState() { public void testDetermineAffectedDataStreams() { Metadata.Builder builder = Metadata.builder(); - DataStream dataStreamWithoutLifecycle = DataStreamTestHelper.newInstance( + DataStream dataStreamWithoutLifecycle = newDataStreamInstance( "ds-no-lifecycle", List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))), 1, @@ -140,7 +141,7 @@ public void testDetermineAffectedDataStreams() { ); builder.put(dataStreamWithoutLifecycle); String dataStreamNoRetention = "ds-no-retention"; - DataStream dataStreamWithLifecycleNoRetention = DataStreamTestHelper.newInstance( + DataStream dataStreamWithLifecycleNoRetention = newDataStreamInstance( dataStreamNoRetention, List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))), 1, @@ -151,7 +152,7 @@ public void testDetermineAffectedDataStreams() { ); builder.put(dataStreamWithLifecycleNoRetention); - DataStream dataStreamWithLifecycleShortRetention = DataStreamTestHelper.newInstance( + DataStream dataStreamWithLifecycleShortRetention = newDataStreamInstance( "ds-no-short-retention", List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))), 1, @@ -162,7 +163,7 @@ public void testDetermineAffectedDataStreams() { ); builder.put(dataStreamWithLifecycleShortRetention); String dataStreamLongRetention = "ds-long-retention"; - DataStream dataStreamWithLifecycleLongRetention = DataStreamTestHelper.newInstance( + DataStream dataStreamWithLifecycleLongRetention = newDataStreamInstance( dataStreamLongRetention, List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))), 1, @@ -191,25 +192,45 @@ public void testDetermineAffectedDataStreams() { { var globalRetention = new DataStreamGlobalRetention(TimeValue.timeValueDays(randomIntBetween(1, 10)), null); var affectedDataStreams = service.determineAffectedDataStreams(globalRetention, clusterState); - assertThat(affectedDataStreams.size(), is(1)); - var dataStream = affectedDataStreams.get(0); - assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention)); - assertThat(dataStream.previousEffectiveRetention(), nullValue()); - assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getDefaultRetention())); + if (dataStreamWithLifecycleNoRetention.isSystem()) { + assertThat(affectedDataStreams.size(), is(0)); + } else { + assertThat(affectedDataStreams.size(), is(1)); + var dataStream = affectedDataStreams.get(0); + assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention)); + assertThat(dataStream.previousEffectiveRetention(), nullValue()); + assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getDefaultRetention())); + } } // Max retention in effect { var globalRetention = new DataStreamGlobalRetention(null, TimeValue.timeValueDays(randomIntBetween(10, 90))); var affectedDataStreams = service.determineAffectedDataStreams(globalRetention, clusterState); - assertThat(affectedDataStreams.size(), is(2)); - var dataStream = affectedDataStreams.get(0); - assertThat(dataStream.dataStreamName(), equalTo(dataStreamLongRetention)); - assertThat(dataStream.previousEffectiveRetention(), notNullValue()); - assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention())); - dataStream = affectedDataStreams.get(1); - assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention)); - assertThat(dataStream.previousEffectiveRetention(), nullValue()); - assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention())); + if (dataStreamWithLifecycleLongRetention.isSystem() && dataStreamWithLifecycleNoRetention.isSystem()) { + assertThat(affectedDataStreams.size(), is(0)); + } else if (dataStreamWithLifecycleLongRetention.isSystem() == false && dataStreamWithLifecycleNoRetention.isSystem() == false) { + assertThat(affectedDataStreams.size(), is(2)); + var dataStream = affectedDataStreams.get(0); + assertThat(dataStream.dataStreamName(), equalTo(dataStreamLongRetention)); + assertThat(dataStream.previousEffectiveRetention(), notNullValue()); + assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention())); + dataStream = affectedDataStreams.get(1); + assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention)); + assertThat(dataStream.previousEffectiveRetention(), nullValue()); + assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention())); + } else if (dataStreamWithLifecycleLongRetention.isSystem() == false) { + assertThat(affectedDataStreams.size(), is(1)); + var dataStream = affectedDataStreams.get(0); + assertThat(dataStream.dataStreamName(), equalTo(dataStreamLongRetention)); + assertThat(dataStream.previousEffectiveRetention(), notNullValue()); + assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention())); + } else { + assertThat(affectedDataStreams.size(), is(1)); + var dataStream = affectedDataStreams.get(0); + assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention)); + assertThat(dataStream.previousEffectiveRetention(), nullValue()); + assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention())); + } } // Requested global retention match the factory retention, so no affected data streams @@ -225,6 +246,29 @@ public void testDetermineAffectedDataStreams() { } } + private static DataStream newDataStreamInstance( + String name, + List indices, + long generation, + Map metadata, + boolean replicated, + @Nullable DataStreamLifecycle lifecycle, + List failureStores + ) { + DataStream.Builder builder = DataStream.builder(name, indices) + .setGeneration(generation) + .setMetadata(metadata) + .setReplicated(replicated) + .setLifecycle(lifecycle) + .setFailureStoreEnabled(failureStores.isEmpty() == false) + .setFailureIndices(failureStores); + if (randomBoolean()) { + builder.setSystem(true); + builder.setHidden(true); + } + return builder.build(); + } + private static DataStreamGlobalRetention randomNonEmptyGlobalRetention() { boolean withDefault = randomBoolean(); return new DataStreamGlobalRetention( diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 707da53b69e51..fb7458662edf7 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -188,6 +188,7 @@ static TransportVersion def(int id) { public static final TransportVersion SECURITY_ROLE_MAPPINGS_IN_CLUSTER_STATE = def(8_647_00_0); public static final TransportVersion ESQL_REQUEST_TABLES = def(8_648_00_0); public static final TransportVersion ROLE_REMOTE_CLUSTER_PRIVS = def(8_649_00_0); + public static final TransportVersion NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS = def(8_650_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 01ce7cbd3346b..1517b368e21ea 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -346,7 +346,8 @@ public XContentBuilder toXContent( } if (dataStream.getLifecycle() != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration, globalRetention); + dataStream.getLifecycle() + .toXContent(builder, params, rolloverConfiguration, dataStream.isSystem() ? null : globalRetention); } if (ilmPolicyName != null) { builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java index bb6c3f90f1b0a..32be73a7b0960 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.datastreams.lifecycle; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -44,6 +45,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj private final String index; private final boolean managedByLifecycle; + private final boolean isSystemDataStream; @Nullable private final Long indexCreationDate; @Nullable @@ -59,6 +61,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj public ExplainIndexDataStreamLifecycle( String index, boolean managedByLifecycle, + boolean isSystemDataStream, @Nullable Long indexCreationDate, @Nullable Long rolloverDate, @Nullable TimeValue generationDate, @@ -67,6 +70,7 @@ public ExplainIndexDataStreamLifecycle( ) { this.index = index; this.managedByLifecycle = managedByLifecycle; + this.isSystemDataStream = isSystemDataStream; this.indexCreationDate = indexCreationDate; this.rolloverDate = rolloverDate; this.generationDateMillis = generationDate == null ? null : generationDate.millis(); @@ -77,6 +81,11 @@ public ExplainIndexDataStreamLifecycle( public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException { this.index = in.readString(); this.managedByLifecycle = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { + this.isSystemDataStream = in.readBoolean(); + } else { + this.isSystemDataStream = false; + } if (managedByLifecycle) { this.indexCreationDate = in.readOptionalLong(); this.rolloverDate = in.readOptionalLong(); @@ -132,7 +141,7 @@ public XContentBuilder toXContent( } if (this.lifecycle != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); + lifecycle.toXContent(builder, params, rolloverConfiguration, isSystemDataStream ? null : globalRetention); } if (this.error != null) { if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) { @@ -151,6 +160,9 @@ public XContentBuilder toXContent( public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeBoolean(managedByLifecycle); + if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { + out.writeBoolean(isSystemDataStream); + } if (managedByLifecycle) { out.writeOptionalLong(indexCreationDate); out.writeOptionalLong(rolloverDate); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index c7384e7003963..d0bc1ee9dc011 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -137,22 +137,30 @@ public Request includeDefaults(boolean includeDefaults) { public static class Response extends ActionResponse implements ChunkedToXContentObject { public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams"); - public record DataStreamLifecycle(String dataStreamName, @Nullable org.elasticsearch.cluster.metadata.DataStreamLifecycle lifecycle) - implements - Writeable, - ToXContentObject { + public record DataStreamLifecycle( + String dataStreamName, + @Nullable org.elasticsearch.cluster.metadata.DataStreamLifecycle lifecycle, + boolean isSystemDataStream + ) implements Writeable, ToXContentObject { public static final ParseField NAME_FIELD = new ParseField("name"); public static final ParseField LIFECYCLE_FIELD = new ParseField("lifecycle"); DataStreamLifecycle(StreamInput in) throws IOException { - this(in.readString(), in.readOptionalWriteable(org.elasticsearch.cluster.metadata.DataStreamLifecycle::new)); + this( + in.readString(), + in.readOptionalWriteable(org.elasticsearch.cluster.metadata.DataStreamLifecycle::new), + in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS) && in.readBoolean() + ); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(dataStreamName); out.writeOptionalWriteable(lifecycle); + if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { + out.writeBoolean(isSystemDataStream); + } } @Override @@ -178,7 +186,7 @@ public XContentBuilder toXContent( builder, org.elasticsearch.cluster.metadata.DataStreamLifecycle.maybeAddEffectiveRetentionParams(params), rolloverConfiguration, - globalRetention + isSystemDataStream ? null : globalRetention ); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 33dab20a81494..16ad072f271ff 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -765,12 +765,14 @@ public List getIndicesPastRetention( LongSupplier nowSupplier, DataStreamGlobalRetention globalRetention ) { - if (lifecycle == null || lifecycle.isEnabled() == false || lifecycle.getEffectiveDataRetention(globalRetention) == null) { + if (lifecycle == null + || lifecycle.isEnabled() == false + || lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention) == null) { return List.of(); } List indicesPastRetention = getNonWriteIndicesOlderThan( - lifecycle.getEffectiveDataRetention(globalRetention), + lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention), indexMetadataSupplier, this::isIndexManagedByDataStreamLifecycle, nowSupplier @@ -1150,7 +1152,7 @@ public XContentBuilder toXContent( } if (lifecycle != null) { builder.field(LIFECYCLE.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); + lifecycle.toXContent(builder, params, rolloverConfiguration, isSystem() ? null : globalRetention); } builder.field(ROLLOVER_ON_WRITE_FIELD.getPreferredName(), rolloverOnWrite); if (autoShardingEvent != null) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index d816da900a083..3fb5e92cb3359 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -145,7 +145,10 @@ public boolean isEnabled() { } /** - * The least amount of time data should be kept by elasticsearch. + * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for + * example, when evaluating the effective retention for a system data stream or a template) then null should be given for + * globalRetention. + * @param globalRetention The global retention, or null if global retention does not exist or should not be applied * @return the time period or null, null represents that data should never be deleted. */ @Nullable @@ -154,7 +157,10 @@ public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention g } /** - * The least amount of time data should be kept by elasticsearch. + * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for + * example, when evaluating the effective retention for a system data stream or a template) then null should be given for + * globalRetention. + * @param globalRetention The global retention, or null if global retention does not exist or should not be applied * @return A tuple containing the time period or null as v1 (where null represents that data should never be deleted), and the non-null * retention source as v2. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 20afd7f9eb3ed..a018f3d93a9bc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -208,12 +208,17 @@ static ClusterState modifyDataStream( ClusterState updateDataLifecycle(ClusterState currentState, List dataStreamNames, @Nullable DataStreamLifecycle lifecycle) { Metadata metadata = currentState.metadata(); Metadata.Builder builder = Metadata.builder(metadata); + boolean atLeastOneDataStreamIsNotSystem = false; for (var dataStreamName : dataStreamNames) { var dataStream = validateDataStream(metadata, dataStreamName); builder.put(dataStream.copy().setLifecycle(lifecycle).build()); + atLeastOneDataStreamIsNotSystem = atLeastOneDataStreamIsNotSystem || dataStream.isSystem() == false; } if (lifecycle != null) { - lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionResolver.resolve(currentState)); + if (atLeastOneDataStreamIsNotSystem) { + // We don't issue any warnings if all data streams are system data streams + lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionResolver.resolve(currentState)); + } } return ClusterState.builder(currentState).metadata(builder.build()).build(); } diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java new file mode 100644 index 0000000000000..285a41f976393 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class GetDataStreamActionTests extends ESTestCase { + + @SuppressWarnings("unchecked") + public void testDataStreamInfoToXContent() throws IOException { + TimeValue configuredRetention = TimeValue.timeValueDays(100); + TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); + TimeValue globalMaxRetention = TimeValue.timeValueDays(50); + + { + // Since this is a system data stream, we expect the global retention to be ignored + boolean isSystem = true; + GetDataStreamAction.Response.DataStreamInfo dataStreamInfo = newDataStreamInfo(isSystem, configuredRetention); + Map resultMap = getXContentMap(dataStreamInfo, globalDefaultRetention, globalMaxRetention); + assertThat(resultMap.get("hidden"), equalTo(true)); + assertThat(resultMap.get("system"), equalTo(true)); + Map lifecycleResult = (Map) resultMap.get("lifecycle"); + assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("effective_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("retention_determined_by"), equalTo("data_stream_configuration")); + } + { + // Since this is not a system data stream, we expect the global retention to override the configured retention + boolean isSystem = false; + GetDataStreamAction.Response.DataStreamInfo dataStreamInfo = newDataStreamInfo(isSystem, configuredRetention); + Map resultMap = getXContentMap(dataStreamInfo, globalDefaultRetention, globalMaxRetention); + assertThat(resultMap.get("hidden"), equalTo(false)); + assertThat(resultMap.get("system"), equalTo(false)); + Map lifecycleResult = (Map) resultMap.get("lifecycle"); + assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep())); + assertThat(lifecycleResult.get("retention_determined_by"), equalTo("max_global_retention")); + } + } + + /* + * Calls toXContent on the given dataStreamInfo, and converts the response to a Map + */ + private Map getXContentMap( + GetDataStreamAction.Response.DataStreamInfo dataStreamInfo, + TimeValue globalDefaultRetention, + TimeValue globalMaxRetention + ) throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + ToXContent.Params params = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS); + RolloverConfiguration rolloverConfiguration = null; + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); + dataStreamInfo.toXContent(builder, params, rolloverConfiguration, globalRetention); + String serialized = Strings.toString(builder); + return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean()); + } + } + + private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boolean isSystem, TimeValue retention) { + DataStream dataStream = newDataStreamInstance(isSystem, retention); + return new GetDataStreamAction.Response.DataStreamInfo( + dataStream, + randomFrom(ClusterHealthStatus.values()), + null, + null, + null, + Map.of(), + randomBoolean() + ); + } + + private static DataStream newDataStreamInstance(boolean isSystem, TimeValue retention) { + List indices = List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))); + DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(retention), null, null); + return DataStream.builder(randomAlphaOfLength(50), indices) + .setGeneration(randomLongBetween(1, 1000)) + .setMetadata(Map.of()) + .setSystem(isSystem) + .setHidden(isSystem) + .setReplicated(randomBoolean()) + .setLifecycle(lifecycle) + .build(); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleResponseTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleResponseTests.java index a47eca7692842..8e920e618e7c5 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleResponseTests.java @@ -204,6 +204,7 @@ public void testToXContent() throws IOException { ExplainIndexDataStreamLifecycle explainIndexWithNullGenerationDate = new ExplainIndexDataStreamLifecycle( index, true, + randomBoolean(), now, randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null, null, @@ -263,6 +264,7 @@ private static ExplainIndexDataStreamLifecycle createRandomIndexDataStreamLifecy return new ExplainIndexDataStreamLifecycle( index, true, + randomBoolean(), now, randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null, randomBoolean() ? TimeValue.timeValueMillis(now) : null, diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java index 7087b677673e7..7f202a6258082 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java @@ -8,14 +8,23 @@ package org.elasticsearch.action.datastreams.lifecycle; +import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.Map; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -27,6 +36,7 @@ public void testGetGenerationTime() { ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( randomAlphaOfLengthBetween(10, 30), true, + randomBoolean(), now, randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null, null, @@ -44,6 +54,7 @@ public void testGetGenerationTime() { explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( randomAlphaOfLengthBetween(10, 30), true, + randomBoolean(), now, randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null, TimeValue.timeValueMillis(now + 100), @@ -64,6 +75,7 @@ public void testGetGenerationTime() { ExplainIndexDataStreamLifecycle indexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( "my-index", false, + randomBoolean(), null, null, null, @@ -78,6 +90,7 @@ public void testGetGenerationTime() { ExplainIndexDataStreamLifecycle indexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( "my-index", true, + randomBoolean(), now, now + 80L, // rolled over in the future (clocks are funny that way) TimeValue.timeValueMillis(now + 100L), @@ -105,6 +118,7 @@ public void testGetTimeSinceIndexCreation() { ExplainIndexDataStreamLifecycle indexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( "my-index", false, + randomBoolean(), null, null, null, @@ -119,6 +133,7 @@ public void testGetTimeSinceIndexCreation() { ExplainIndexDataStreamLifecycle indexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( "my-index", true, + randomBoolean(), now + 80L, // created in the future (clocks are funny that way) null, null, @@ -153,6 +168,7 @@ public void testGetTimeSinceRollover() { ExplainIndexDataStreamLifecycle indexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( "my-index", false, + randomBoolean(), null, null, null, @@ -167,6 +183,7 @@ public void testGetTimeSinceRollover() { ExplainIndexDataStreamLifecycle indexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( "my-index", true, + randomBoolean(), now - 50L, now + 100L, // rolled over in the future TimeValue.timeValueMillis(now), @@ -177,6 +194,62 @@ public void testGetTimeSinceRollover() { } } + @SuppressWarnings("unchecked") + public void testToXContent() throws Exception { + TimeValue configuredRetention = TimeValue.timeValueDays(100); + TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); + TimeValue globalMaxRetention = TimeValue.timeValueDays(50); + DataStreamLifecycle dataStreamLifecycle = new DataStreamLifecycle( + new DataStreamLifecycle.Retention(configuredRetention), + null, + null + ); + { + boolean isSystemDataStream = true; + ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = createManagedIndexDataStreamLifecycleExplanation( + System.currentTimeMillis(), + dataStreamLifecycle, + isSystemDataStream + ); + Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention); + Map lifecycleResult = (Map) resultMap.get("lifecycle"); + assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("effective_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("retention_determined_by"), equalTo("data_stream_configuration")); + } + { + boolean isSystemDataStream = false; + ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = createManagedIndexDataStreamLifecycleExplanation( + System.currentTimeMillis(), + dataStreamLifecycle, + isSystemDataStream + ); + Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention); + Map lifecycleResult = (Map) resultMap.get("lifecycle"); + assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep())); + assertThat(lifecycleResult.get("retention_determined_by"), equalTo("max_global_retention")); + } + } + + /* + * Calls toXContent on the given explainIndexDataStreamLifecycle, and converts the response to a Map + */ + private Map getXContentMap( + ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle, + TimeValue globalDefaultRetention, + TimeValue globalMaxRetention + ) throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + ToXContent.Params params = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS); + RolloverConfiguration rolloverConfiguration = null; + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); + explainIndexDataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); + String serialized = Strings.toString(builder); + return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean()); + } + } + @Override protected Writeable.Reader instanceReader() { return ExplainIndexDataStreamLifecycle::new; @@ -195,10 +268,19 @@ protected ExplainIndexDataStreamLifecycle mutateInstance(ExplainIndexDataStreamL private static ExplainIndexDataStreamLifecycle createManagedIndexDataStreamLifecycleExplanation( long now, @Nullable DataStreamLifecycle lifecycle + ) { + return createManagedIndexDataStreamLifecycleExplanation(now, lifecycle, randomBoolean()); + } + + private static ExplainIndexDataStreamLifecycle createManagedIndexDataStreamLifecycleExplanation( + long now, + @Nullable DataStreamLifecycle lifecycle, + boolean isSystemDataStream ) { return new ExplainIndexDataStreamLifecycle( randomAlphaOfLengthBetween(10, 30), true, + isSystemDataStream, now, randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null, TimeValue.timeValueMillis(now), diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java new file mode 100644 index 0000000000000..c769e504ef15b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.datastreams.lifecycle; + +import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; +import static org.hamcrest.Matchers.equalTo; + +public class GetDataStreamLifecycleActionTests extends ESTestCase { + + @SuppressWarnings("unchecked") + public void testDataStreamLifecycleToXContent() throws Exception { + TimeValue configuredRetention = TimeValue.timeValueDays(100); + TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); + TimeValue globalMaxRetention = TimeValue.timeValueDays(50); + DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(configuredRetention), null, null); + { + boolean isSystemDataStream = true; + GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle( + lifecycle, + isSystemDataStream + ); + Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention); + Map lifecycleResult = (Map) resultMap.get("lifecycle"); + assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("effective_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("retention_determined_by"), equalTo("data_stream_configuration")); + } + { + boolean isSystemDataStream = false; + GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle( + lifecycle, + isSystemDataStream + ); + Map resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention); + Map lifecycleResult = (Map) resultMap.get("lifecycle"); + assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); + assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep())); + assertThat(lifecycleResult.get("retention_determined_by"), equalTo("max_global_retention")); + } + } + + private GetDataStreamLifecycleAction.Response.DataStreamLifecycle createDataStreamLifecycle( + DataStreamLifecycle lifecycle, + boolean isSystemDataStream + ) { + return new GetDataStreamLifecycleAction.Response.DataStreamLifecycle(randomAlphaOfLength(50), lifecycle, isSystemDataStream); + } + + /* + * Calls toXContent on the given dataStreamLifecycle, and converts the response to a Map + */ + private Map getXContentMap( + GetDataStreamLifecycleAction.Response.DataStreamLifecycle dataStreamLifecycle, + TimeValue globalDefaultRetention, + TimeValue globalMaxRetention + ) throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + ToXContent.Params params = new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless")); + RolloverConfiguration rolloverConfiguration = null; + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); + dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); + String serialized = Strings.toString(builder); + return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 083d6e651dd1e..d42b6096b6e32 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -1760,14 +1760,15 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws } DataStreamLifecycle lifecycle = new DataStreamLifecycle(); + boolean isSystem = randomBoolean(); DataStream dataStream = new DataStream( dataStreamName, indices, generation, metadata, + isSystem, randomBoolean(), - randomBoolean(), - false, // Some tests don't work well with system data streams, since these data streams require special handling + isSystem, System::currentTimeMillis, randomBoolean(), randomBoolean() ? IndexMode.STANDARD : null, // IndexMode.TIME_SERIES triggers validation that many unit tests doesn't pass @@ -1794,7 +1795,8 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws } // We check that even if there was no retention provided by the user, the global retention applies assertThat(serialized, not(containsString("data_retention"))); - if (globalRetention.getDefaultRetention() != null || globalRetention.getMaxRetention() != null) { + if (dataStream.isSystem() == false + && (globalRetention.getDefaultRetention() != null || globalRetention.getMaxRetention() != null)) { assertThat(serialized, containsString("effective_retention")); } else { assertThat(serialized, not(containsString("effective_retention")));