diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index f20a3393c191c..1bda7d8959bee 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -130,14 +130,18 @@ The response will look like the following: "name": "my-data-stream-1", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" } }, { "name": "my-data-stream-2", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" } } ] diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc index 5670faaade3ce..7be2b30b9b83c 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc @@ -74,7 +74,9 @@ The response will look like: "generation_time": "6.84s", <9> "lifecycle": { "enabled": true, - "data_retention": "30d" <10> + "data_retention": "30d", + "effective_retention": "30d" <10> + "retention_determined_by": "data_stream_configuration" } } } diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc index 6f1d81ab6ead2..ecfdc16884082 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc @@ -93,10 +93,12 @@ The result will look like this: { "data_streams": [ { - "name": "my-data-stream",<1> + "name": "my-data-stream", <1> "lifecycle": { - "enabled": true, <2> - "data_retention": "7d" <3> + "enabled": true, <2> + "data_retention": "7d", <3> + "effective_retention": "7d", <4> + "retention_determined_by": "data_stream_configuration" <5> } } ] @@ -104,8 +106,11 @@ The result will look like this: -------------------------------------------------- <1> The name of your data stream. <2> Shows if the data stream lifecycle is enabled for this data stream. -<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will +<3> The desired retention period of the data indexed in this data stream, this means that if there are no other limitations +the data for this data stream will be preserved for at least 7 days. +<4> The effective retention, this means that the data in this data stream will be kept at least for 7 days. After that {es} can delete it at its own discretion. +<5> The configuration that determined the effective retention. If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the <>: @@ -128,7 +133,9 @@ The result will look like this: "time_since_index_creation": "1.6m", <3> "lifecycle": { <4> "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" } } } diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 3125c82120d8d..65eaf472890f4 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -200,10 +200,10 @@ PUT _index_template/dsl-data-stream-template "template": { "settings": { "index.lifecycle.name": "pre-dsl-ilm-policy", - "index.lifecycle.prefer_ilm": false <1> + "index.lifecycle.prefer_ilm": false <1> }, - "lifecycle": { - "data_retention": "7d" <2> + "lifecycle": { <2> + "data_retention": "7d" <3> } } } @@ -215,6 +215,8 @@ PUT _index_template/dsl-data-stream-template precedence over data stream lifecycle. <2> We're configuring the data stream lifecycle so _new_ data streams will be managed by data stream lifecycle. +<3> The desired retention, meaning that this data stream should keep the data for at least 7 days, +if this retention is possible. We've now made sure that new data streams will be managed by data stream lifecycle. @@ -268,7 +270,9 @@ GET _data_stream/dsl-data-stream "template": "dsl-data-stream-template", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" }, "ilm_policy": "pre-dsl-ilm-policy", "next_generation_managed_by": "Data stream lifecycle", <3> @@ -346,7 +350,9 @@ GET _data_stream/dsl-data-stream "template": "dsl-data-stream-template", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" }, "ilm_policy": "pre-dsl-ilm-policy", "next_generation_managed_by": "Data stream lifecycle", diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java index e44ee5107711f..2b1d4ae01f565 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -199,7 +200,8 @@ static GetDataStreamAction.Response innerOperation( } return new GetDataStreamAction.Response( dataStreamInfos, - request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null + request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null, + DataStreamGlobalRetention.getFromClusterState(state) ); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java index 676052f76d564..5bfdf2d382005 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.datastreams.lifecycle.action; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -16,6 +17,8 @@ import org.elasticsearch.action.datastreams.lifecycle.ExplainIndexDataStreamLifecycle; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -136,23 +139,33 @@ public Request indicesOptions(IndicesOptions indicesOptions) { } /** - * Class representing the response for the explain of the data stream lifecycle action for one or more indices. + * Class representing the response for the 'explain' of the data stream lifecycle action for one or more indices. */ public static class Response extends ActionResponse implements ChunkedToXContentObject { public static final ParseField INDICES_FIELD = new ParseField("indices"); - private List indices; + private final List indices; @Nullable private final RolloverConfiguration rolloverConfiguration; + @Nullable + private final DataStreamGlobalRetention globalRetention; - public Response(List indices, @Nullable RolloverConfiguration rolloverConfiguration) { + public Response( + List indices, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) { this.indices = indices; this.rolloverConfiguration = rolloverConfiguration; + this.globalRetention = globalRetention; } public Response(StreamInput in) throws IOException { super(in); this.indices = in.readCollectionAsList(ExplainIndexDataStreamLifecycle::new); this.rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new); + this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + ? in.readOptionalWriteable(DataStreamGlobalRetention::read) + : null; } public List getIndices() { @@ -163,10 +176,17 @@ public RolloverConfiguration getRolloverConfiguration() { return rolloverConfiguration; } + public DataStreamGlobalRetention getGlobalRetention() { + return globalRetention; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeCollection(indices); out.writeOptionalWriteable(rolloverConfiguration); + if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeOptionalWriteable(globalRetention); + } } @Override @@ -178,12 +198,14 @@ public boolean equals(Object o) { return false; } Response response = (Response) o; - return Objects.equals(indices, response.indices) && Objects.equals(rolloverConfiguration, response.rolloverConfiguration); + return Objects.equals(indices, response.indices) + && Objects.equals(rolloverConfiguration, response.rolloverConfiguration) + && Objects.equals(globalRetention, response.globalRetention); } @Override public int hashCode() { - return Objects.hash(indices, rolloverConfiguration); + return Objects.hash(indices, rolloverConfiguration, globalRetention); } @Override @@ -194,7 +216,11 @@ public Iterator toXContentChunked(ToXContent.Params outerP return builder; }), Iterators.map(indices.iterator(), explainIndexDataLifecycle -> (builder, params) -> { builder.field(explainIndexDataLifecycle.getIndex()); - explainIndexDataLifecycle.toXContent(builder, params, rolloverConfiguration); + ToXContent.Params withEffectiveRetentionParams = new ToXContent.DelegatingMapParams( + DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, + params + ); + explainIndexDataLifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); return builder; }), Iterators.single((builder, params) -> { builder.endObject(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java index 8149e1a0df443..79e1b71771559 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.datastreams.lifecycle.action; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -14,6 +15,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -157,19 +159,24 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** - * Converts the response to XContent and passes the RolloverConditions, when provided, to the data stream lifecycle. + * Converts the response to XContent and passes the RolloverConditions and the global retention, when provided, + * to the data stream lifecycle. */ - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.field(NAME_FIELD.getPreferredName(), dataStreamName); if (lifecycle != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration); + lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); } builder.endObject(); return builder; @@ -179,18 +186,31 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla private final List dataStreamLifecycles; @Nullable private final RolloverConfiguration rolloverConfiguration; + @Nullable + private final DataStreamGlobalRetention globalRetention; public Response(List dataStreamLifecycles) { - this(dataStreamLifecycles, null); + this(dataStreamLifecycles, null, null); } - public Response(List dataStreamLifecycles, @Nullable RolloverConfiguration rolloverConfiguration) { + public Response( + List dataStreamLifecycles, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) { this.dataStreamLifecycles = dataStreamLifecycles; this.rolloverConfiguration = rolloverConfiguration; + this.globalRetention = globalRetention; } public Response(StreamInput in) throws IOException { - this(in.readCollectionAsList(Response.DataStreamLifecycle::new), in.readOptionalWriteable(RolloverConfiguration::new)); + this( + in.readCollectionAsList(Response.DataStreamLifecycle::new), + in.readOptionalWriteable(RolloverConfiguration::new), + in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + ? in.readOptionalWriteable(DataStreamGlobalRetention::read) + : null + ); } public List getDataStreamLifecycles() { @@ -206,6 +226,9 @@ public RolloverConfiguration getRolloverConfiguration() { public void writeTo(StreamOutput out) throws IOException { out.writeCollection(dataStreamLifecycles); out.writeOptionalWriteable(rolloverConfiguration); + if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeOptionalWriteable(globalRetention); + } } @Override @@ -214,17 +237,17 @@ public Iterator toXContentChunked(ToXContent.Params outerParams) { builder.startObject(); builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); return builder; - }), - Iterators.map( - dataStreamLifecycles.iterator(), - dataStreamLifecycle -> (builder, params) -> dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration) - ), - Iterators.single((builder, params) -> { - builder.endArray(); - builder.endObject(); - return builder; - }) - ); + }), Iterators.map(dataStreamLifecycles.iterator(), dataStreamLifecycle -> (builder, params) -> { + ToXContent.Params withEffectiveRetentionParams = new ToXContent.DelegatingMapParams( + org.elasticsearch.cluster.metadata.DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, + params + ); + return dataStreamLifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + }), Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + })); } @Override @@ -233,12 +256,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Response response = (Response) o; return dataStreamLifecycles.equals(response.dataStreamLifecycles) - && Objects.equals(rolloverConfiguration, response.rolloverConfiguration); + && Objects.equals(rolloverConfiguration, response.rolloverConfiguration) + && Objects.equals(globalRetention, response.globalRetention); } @Override public int hashCode() { - return Objects.hash(dataStreamLifecycles, rolloverConfiguration); + return Objects.hash(dataStreamLifecycles, rolloverConfiguration, globalRetention); } } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java index a42e8dfefc468..a5c3b092a8913 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -111,7 +112,8 @@ protected void masterOperation( listener.onResponse( new ExplainDataStreamLifecycleAction.Response( explainIndices, - request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null + request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null, + DataStreamGlobalRetention.getFromClusterState(state) ) ); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java index 29b88fc5748bf..84144cdcb0379 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -89,7 +90,8 @@ protected void masterOperation( ) .sorted(Comparator.comparing(GetDataStreamLifecycleAction.Response.DataStreamLifecycle::dataStreamName)) .toList(), - request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null + request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null, + DataStreamGlobalRetention.getFromClusterState(state) ) ); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java index 637fb44affb6f..2a356e3ebb166 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java @@ -11,11 +11,13 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -35,6 +37,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; public class GetDataStreamsTransportActionTests extends ESTestCase { @@ -248,4 +251,45 @@ public void testGetTimeSeriesMixedDataStream() { ) ); } + + public void testPassingGlobalRetention() { + ClusterState state; + { + var mBuilder = new Metadata.Builder(); + DataStreamTestHelper.getClusterStateWithDataStreams( + mBuilder, + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + false + ); + state = ClusterState.builder(new ClusterName("_name")).metadata(mBuilder).build(); + } + + var req = new GetDataStreamAction.Request(new String[] {}); + var response = GetDataStreamsTransportAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings() + ); + assertThat(response.getGlobalRetention(), nullValue()); + DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( + TimeValue.timeValueDays(randomIntBetween(1, 5)), + TimeValue.timeValueDays(randomIntBetween(5, 10)) + ); + state = ClusterState.builder(state).putCustom(DataStreamGlobalRetention.TYPE, globalRetention).build(); + response = GetDataStreamsTransportAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings() + ); + assertThat(response.getGlobalRetention(), equalTo(globalRetention)); + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java index 829fe454f7463..462c0626c6296 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.datastreams.lifecycle.ErrorEntry; import org.elasticsearch.action.datastreams.lifecycle.ExplainIndexDataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; @@ -35,6 +37,7 @@ import static org.elasticsearch.datastreams.lifecycle.action.ExplainDataStreamLifecycleAction.Response; import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -66,7 +69,7 @@ public void testToXContent() throws IOException { ExplainIndexDataStreamLifecycle explainIndex = createRandomIndexDataStreamLifecycleExplanation(now, lifecycle); explainIndex.setNowSupplier(() -> now); { - Response response = new Response(List.of(explainIndex), null); + Response response = new Response(List.of(explainIndex), null, null); XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); response.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xcontent -> { @@ -103,7 +106,7 @@ public void testToXContent() throws IOException { } else { assertThat(explainIndexMap.get("generation_time"), is(nullValue())); } - assertThat(explainIndexMap.get("lifecycle"), is(Map.of("enabled", true))); // empty lifecycle + assertThat(explainIndexMap.get("lifecycle"), is(Map.of("enabled", true))); if (explainIndex.getError() != null) { Map errorObject = (Map) explainIndexMap.get("error"); assertThat(errorObject.get(ErrorEntry.MESSAGE_FIELD.getPreferredName()), is(explainIndex.getError().error())); @@ -132,7 +135,11 @@ public void testToXContent() throws IOException { new MinPrimaryShardDocsCondition(4L) ) ); - Response response = new Response(List.of(explainIndex), new RolloverConfiguration(rolloverConditions)); + Response response = new Response( + List.of(explainIndex), + new RolloverConfiguration(rolloverConditions), + DataStreamTestHelper.randomGlobalRetention() + ); XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); response.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xcontent -> { @@ -186,9 +193,27 @@ public void testToXContent() throws IOException { assertThat(explainIndexMap.get("error"), is(nullValue())); } - Map lifecycleRollover = (Map) ((Map) explainIndexMap.get("lifecycle")).get( - "rollover" - ); + Map lifecycleMap = (Map) explainIndexMap.get("lifecycle"); + assertThat(lifecycleMap.get("data_retention"), nullValue()); + + if (response.getGlobalRetention() == null) { + assertThat(lifecycleMap.get("effective_retention"), nullValue()); + assertThat(lifecycleMap.get("retention_determined_by"), nullValue()); + } else if (response.getGlobalRetention().getDefaultRetention() != null) { + assertThat( + lifecycleMap.get("effective_retention"), + equalTo(response.getGlobalRetention().getDefaultRetention().getStringRep()) + ); + assertThat(lifecycleMap.get("retention_determined_by"), equalTo("default_global_retention")); + } else { + assertThat( + lifecycleMap.get("effective_retention"), + equalTo(response.getGlobalRetention().getMaxRetention().getStringRep()) + ); + assertThat(lifecycleMap.get("retention_determined_by"), equalTo("max_global_retention")); + } + + Map lifecycleRollover = (Map) lifecycleMap.get("rollover"); assertThat(lifecycleRollover.get("min_primary_shard_docs"), is(4)); assertThat(lifecycleRollover.get("max_primary_shard_docs"), is(9)); } @@ -212,7 +237,7 @@ public void testToXContent() throws IOException { ) : null ); - Response response = new Response(List.of(explainIndexWithNullGenerationDate), null); + Response response = new Response(List.of(explainIndexWithNullGenerationDate), null, null); XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); response.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xcontent -> { @@ -241,6 +266,7 @@ public void testChunkCount() { createRandomIndexDataStreamLifecycleExplanation(now, lifecycle), createRandomIndexDataStreamLifecycleExplanation(now, lifecycle) ), + null, null ); @@ -296,6 +322,12 @@ private Response randomResponse() { Map.of(MaxPrimaryShardDocsCondition.NAME, new MaxPrimaryShardDocsCondition(randomLongBetween(1000, 199_999_000))) ) ) + : null, + randomBoolean() + ? new DataStreamGlobalRetention( + TimeValue.timeValueDays(randomIntBetween(1, 10)), + TimeValue.timeValueDays(randomIntBetween(10, 20)) + ) : null ); } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 9184e46b11542..b5070c5cbd065 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -150,6 +150,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_SERIALIZE_BIG_ARRAY = def(8_610_00_0); public static final TransportVersion AUTO_SHARDING_ROLLOVER_CONDITION = def(8_611_00_0); public static final TransportVersion KNN_QUERY_VECTOR_BUILDER = def(8_612_00_0); + public static final TransportVersion USE_DATA_STREAM_GLOBAL_RETENTION = def(8_613_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java index 7d2dad80bf35a..626feeed161f4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java @@ -15,6 +15,8 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -117,6 +119,8 @@ public static class Response extends ActionResponse implements ToXContentObject private final Map componentTemplates; @Nullable private final RolloverConfiguration rolloverConfiguration; + @Nullable + private final DataStreamGlobalRetention globalRetention; public Response(StreamInput in) throws IOException { super(in); @@ -126,16 +130,27 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read); + } else { + globalRetention = null; + } } public Response(Map componentTemplates) { this.componentTemplates = componentTemplates; this.rolloverConfiguration = null; + this.globalRetention = null; } - public Response(Map componentTemplates, @Nullable RolloverConfiguration rolloverConfiguration) { + public Response( + Map componentTemplates, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) { this.componentTemplates = componentTemplates; this.rolloverConfiguration = rolloverConfiguration; + this.globalRetention = globalRetention; } public Map getComponentTemplates() { @@ -146,12 +161,19 @@ public RolloverConfiguration getRolloverConfiguration() { return rolloverConfiguration; } + public DataStreamGlobalRetention getGlobalRetention() { + return globalRetention; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(componentTemplates, StreamOutput::writeWriteable); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } + if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeOptionalWriteable(globalRetention); + } } @Override @@ -160,23 +182,25 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Response that = (Response) o; return Objects.equals(componentTemplates, that.componentTemplates) - && Objects.equals(rolloverConfiguration, that.rolloverConfiguration); + && Objects.equals(rolloverConfiguration, that.rolloverConfiguration) + && Objects.equals(globalRetention, that.globalRetention); } @Override public int hashCode() { - return Objects.hash(componentTemplates, rolloverConfiguration); + return Objects.hash(componentTemplates, rolloverConfiguration, globalRetention); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); builder.startArray(COMPONENT_TEMPLATES.getPreferredName()); for (Map.Entry componentTemplate : this.componentTemplates.entrySet()) { builder.startObject(); builder.field(NAME.getPreferredName(), componentTemplate.getKey()); builder.field(COMPONENT_TEMPLATE.getPreferredName()); - componentTemplate.getValue().toXContent(builder, params, rolloverConfiguration); + componentTemplate.getValue().toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); builder.endObject(); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java index f75443bad2854..515b8f9fd5c1a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java @@ -15,6 +15,8 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -118,7 +120,10 @@ public static class Response extends ActionResponse implements ToXContentObject public static final ParseField INDEX_TEMPLATE = new ParseField("index_template"); private final Map indexTemplates; + @Nullable private final RolloverConfiguration rolloverConfiguration; + @Nullable + private final DataStreamGlobalRetention globalRetention; public Response(StreamInput in) throws IOException { super(in); @@ -128,16 +133,27 @@ public Response(StreamInput in) throws IOException { } else { rolloverConfiguration = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read); + } else { + globalRetention = null; + } } public Response(Map indexTemplates) { this.indexTemplates = indexTemplates; this.rolloverConfiguration = null; + this.globalRetention = null; } - public Response(Map indexTemplates, RolloverConfiguration rolloverConfiguration) { + public Response( + Map indexTemplates, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) { this.indexTemplates = indexTemplates; this.rolloverConfiguration = rolloverConfiguration; + this.globalRetention = globalRetention; } public Map indexTemplates() { @@ -150,6 +166,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } + if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeOptionalWriteable(globalRetention); + } } @Override @@ -157,23 +176,26 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; GetComposableIndexTemplateAction.Response that = (GetComposableIndexTemplateAction.Response) o; - return Objects.equals(indexTemplates, that.indexTemplates) && Objects.equals(rolloverConfiguration, that.rolloverConfiguration); + return Objects.equals(indexTemplates, that.indexTemplates) + && Objects.equals(rolloverConfiguration, that.rolloverConfiguration) + && Objects.equals(globalRetention, that.globalRetention); } @Override public int hashCode() { - return Objects.hash(indexTemplates, rolloverConfiguration); + return Objects.hash(indexTemplates, rolloverConfiguration, globalRetention); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); builder.startArray(INDEX_TEMPLATES.getPreferredName()); for (Map.Entry indexTemplate : this.indexTemplates.entrySet()) { builder.startObject(); builder.field(NAME.getPreferredName(), indexTemplate.getKey()); builder.field(INDEX_TEMPLATE.getPreferredName()); - indexTemplate.getValue().toXContent(builder, params, rolloverConfiguration); + indexTemplate.getValue().toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); builder.endObject(); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java index e76dc0f46eea2..d081570b2a365 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -96,7 +97,8 @@ protected void masterOperation( listener.onResponse( new GetComponentTemplateAction.Response( results, - clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) + clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING), + DataStreamGlobalRetention.getFromClusterState(state) ) ); } else { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java index c9b2a23c38828..99360d2eb7bf8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -94,7 +95,8 @@ protected void masterOperation( listener.onResponse( new GetComposableIndexTemplateAction.Response( results, - clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) + clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING), + DataStreamGlobalRetention.getFromClusterState(state) ) ); } else { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 106f1a7e4f393..378df2d7d53e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -11,6 +11,8 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -37,27 +39,35 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToX @Nullable // the resolved settings, mappings and aliases for the matched templates, if any - private Template resolvedTemplate; + private final Template resolvedTemplate; @Nullable // a map of template names and their index patterns that would overlap when matching the given index name - private Map> overlappingTemplates; + private final Map> overlappingTemplates; @Nullable - private RolloverConfiguration rolloverConfiguration = null; + private final RolloverConfiguration rolloverConfiguration; + @Nullable + private final DataStreamGlobalRetention globalRetention; - public SimulateIndexTemplateResponse(@Nullable Template resolvedTemplate, @Nullable Map> overlappingTemplates) { - this(resolvedTemplate, overlappingTemplates, null); + public SimulateIndexTemplateResponse( + @Nullable Template resolvedTemplate, + @Nullable Map> overlappingTemplates, + DataStreamGlobalRetention globalRetention + ) { + this(resolvedTemplate, overlappingTemplates, null, globalRetention); } public SimulateIndexTemplateResponse( @Nullable Template resolvedTemplate, @Nullable Map> overlappingTemplates, - @Nullable RolloverConfiguration rolloverConfiguration + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention ) { this.resolvedTemplate = resolvedTemplate; this.overlappingTemplates = overlappingTemplates; this.rolloverConfiguration = rolloverConfiguration; + this.globalRetention = globalRetention; } public SimulateIndexTemplateResponse(StreamInput in) throws IOException { @@ -73,9 +83,12 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException { } else { this.overlappingTemplates = null; } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new); - } + rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) + ? in.readOptionalWriteable(RolloverConfiguration::new) + : null; + globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + ? in.readOptionalWriteable(DataStreamGlobalRetention::read) + : null; } @Override @@ -94,14 +107,18 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } + if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeOptionalWriteable(globalRetention); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); if (this.resolvedTemplate != null) { builder.field(TEMPLATE.getPreferredName()); - this.resolvedTemplate.toXContent(builder, params, rolloverConfiguration); + this.resolvedTemplate.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); } if (this.overlappingTemplates != null) { builder.startArray(OVERLAPPING.getPreferredName()); @@ -127,12 +144,14 @@ public boolean equals(Object o) { } SimulateIndexTemplateResponse that = (SimulateIndexTemplateResponse) o; return Objects.equals(resolvedTemplate, that.resolvedTemplate) - && Objects.deepEquals(overlappingTemplates, that.overlappingTemplates); + && Objects.deepEquals(overlappingTemplates, that.overlappingTemplates) + && Objects.equals(rolloverConfiguration, that.rolloverConfiguration) + && Objects.equals(globalRetention, that.globalRetention); } @Override public int hashCode() { - return Objects.hash(resolvedTemplate, overlappingTemplates); + return Objects.hash(resolvedTemplate, overlappingTemplates, rolloverConfiguration, globalRetention); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index bee6ab7f78be0..51e17999da5c5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -112,6 +113,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { + final DataStreamGlobalRetention globalRetention = DataStreamGlobalRetention.getFromClusterState(state); final ClusterState stateWithTemplate; if (request.getIndexTemplateRequest() != null) { // we'll "locally" add the template defined by the user in the cluster state (as if it existed in the system) @@ -137,7 +139,7 @@ protected void masterOperation( String matchingTemplate = findV2Template(stateWithTemplate.metadata(), request.getIndexName(), false); if (matchingTemplate == null) { - listener.onResponse(new SimulateIndexTemplateResponse(null, null)); + listener.onResponse(new SimulateIndexTemplateResponse(null, null, null)); return; } @@ -165,11 +167,12 @@ protected void masterOperation( new SimulateIndexTemplateResponse( template, overlapping, - clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) + clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING), + globalRetention ) ); } else { - listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); + listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping, globalRetention)); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 1f35d0b8a1268..39cf5f43f39ec 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; @@ -99,6 +100,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { + final DataStreamGlobalRetention globalRetention = DataStreamGlobalRetention.getFromClusterState(state); String uuid = UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT); final String temporaryIndexName = "simulate_template_index_" + uuid; final ClusterState stateWithTemplate; @@ -176,11 +178,12 @@ protected void masterOperation( new SimulateIndexTemplateResponse( template, overlapping, - clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) + clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING), + globalRetention ) ); } else { - listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); + listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping, globalRetention)); } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 8c469f7dffc4d..36f2ff4fffa96 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -19,6 +19,8 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAutoShardingEvent; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -276,14 +278,19 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** - * Converts the response to XContent and passes the RolloverConditions, when provided, to the data stream. + * Converts the response to XContent and passes the RolloverConditions and the global retention, when provided, + * to the data stream. */ - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.field(DataStream.NAME_FIELD.getPreferredName(), dataStream.getName()); builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName()) @@ -339,7 +346,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla } if (dataStream.getLifecycle() != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration); + dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration, globalRetention); } if (ilmPolicyName != null) { builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName); @@ -483,20 +490,30 @@ public void writeTo(StreamOutput out) throws IOException { private final List dataStreams; @Nullable private final RolloverConfiguration rolloverConfiguration; + @Nullable + private final DataStreamGlobalRetention globalRetention; public Response(List dataStreams) { - this(dataStreams, null); + this(dataStreams, null, null); } - public Response(List dataStreams, @Nullable RolloverConfiguration rolloverConfiguration) { + public Response( + List dataStreams, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) { this.dataStreams = dataStreams; this.rolloverConfiguration = rolloverConfiguration; + this.globalRetention = globalRetention; } public Response(StreamInput in) throws IOException { this( in.readCollectionAsList(DataStreamInfo::new), - in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null + in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null, + in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION) + ? in.readOptionalWriteable(DataStreamGlobalRetention::read) + : null ); } @@ -509,20 +526,29 @@ public RolloverConfiguration getRolloverConfiguration() { return rolloverConfiguration; } + @Nullable + public DataStreamGlobalRetention getGlobalRetention() { + return globalRetention; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeCollection(dataStreams); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeOptionalWriteable(rolloverConfiguration); } + if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) { + out.writeOptionalWriteable(globalRetention); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); for (DataStreamInfo dataStream : dataStreams) { - dataStream.toXContent(builder, params, rolloverConfiguration); + dataStream.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); } builder.endArray(); builder.endObject(); @@ -534,12 +560,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Response response = (Response) o; - return dataStreams.equals(response.dataStreams) && Objects.equals(rolloverConfiguration, response.rolloverConfiguration); + return dataStreams.equals(response.dataStreams) + && Objects.equals(rolloverConfiguration, response.rolloverConfiguration) + && Objects.equals(globalRetention, response.globalRetention); } @Override public int hashCode() { - return Objects.hash(dataStreams, rolloverConfiguration); + return Objects.hash(dataStreams, rolloverConfiguration, globalRetention); } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java index 640d8a9efe8ac..2b79377fb71e0 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.datastreams.lifecycle; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -98,11 +99,15 @@ public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.field(INDEX_FIELD.getPreferredName(), index); builder.field(MANAGED_BY_LIFECYCLE_FIELD.getPreferredName(), managedByLifecycle); @@ -127,7 +132,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla } if (this.lifecycle != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration); + Params withEffectiveRetentionParams = new DelegatingMapParams( + DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, + params + ); + lifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); } if (this.error != null) { if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index d3d758e110ff3..a11ec64dc6f2c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -163,17 +163,21 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** * Converts the component template to XContent and passes the RolloverConditions, when provided, to the template. */ - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.field(TEMPLATE.getPreferredName()); - this.template.toXContent(builder, params, rolloverConfiguration); + this.template.toXContent(builder, params, rolloverConfiguration, globalRetention); if (this.version != null) { builder.field(VERSION.getPreferredName(), this.version); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 7702ec0ac0b5c..8e8e6fff4cc6a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -259,19 +259,23 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** * Converts the composable index template to XContent and passes the RolloverConditions, when provided, to the template. */ - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); builder.stringListField(INDEX_PATTERNS.getPreferredName(), this.indexPatterns); if (this.template != null) { builder.field(TEMPLATE.getPreferredName()); - this.template.toXContent(builder, params, rolloverConfiguration); + this.template.toXContent(builder, params, rolloverConfiguration, globalRetention); } if (this.componentTemplates != null) { builder.stringListField(COMPOSED_OF.getPreferredName(), this.componentTemplates); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index a8b094bafde2e..6db7b2cf670bc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -53,6 +54,14 @@ public class DataStreamLifecycle implements SimpleDiffable, public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X; public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode"; + // The following XContent params are used to enrich the DataStreamLifecycle json with effective retention information + // This should be set only when the lifecycle is used in a response to the user and NEVER when we expect the json to + // be deserialized. + public static final String INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME = "include_effective_retention"; + public static final Map INCLUDE_EFFECTIVE_RETENTION_PARAMS = Map.of( + DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, + "true" + ); /** * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that @@ -79,6 +88,8 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static final ParseField ENABLED_FIELD = new ParseField("enabled"); public static final ParseField DATA_RETENTION_FIELD = new ParseField("data_retention"); + public static final ParseField EFFECTIVE_RETENTION_FIELD = new ParseField("effective_retention"); + public static final ParseField RETENTION_SOURCE_FIELD = new ParseField("retention_determined_by"); public static final ParseField DOWNSAMPLING_FIELD = new ParseField("downsampling"); private static final ParseField ROLLOVER_FIELD = new ParseField("rollover"); @@ -130,17 +141,6 @@ public boolean isEnabled() { return enabled; } - /** - * The least amount of time data should be kept by elasticsearch. - * @return the time period or null, null represents that data should never be deleted. - * @deprecated use {@link #getEffectiveDataRetention(DataStreamGlobalRetention)} - */ - @Deprecated - @Nullable - public TimeValue getEffectiveDataRetention() { - return getEffectiveDataRetention(null); - } - /** * The least amount of time data should be kept by elasticsearch. * @return the time period or null, null represents that data should never be deleted. @@ -275,17 +275,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } /** - * Converts the data stream lifecycle to XContent and injects the RolloverConditions if they exist. - * @deprecated use {@link #toXContent(XContentBuilder, Params, RolloverConfiguration, DataStreamGlobalRetention)} - */ - @Deprecated - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { - return toXContent(builder, params, rolloverConfiguration, null); - } - - /** - * Converts the data stream lifecycle to XContent and injects the RolloverConditions and the global retention if they exist. + * Converts the data stream lifecycle to XContent, enriches it with effective retention information when requested + * and injects the RolloverConditions if they exist. + * In order to request the effective retention you need to set {@link #INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME} to true + * in the XContent params. */ public XContentBuilder toXContent( XContentBuilder builder, @@ -302,6 +295,14 @@ public XContentBuilder toXContent( builder.field(DATA_RETENTION_FIELD.getPreferredName(), dataRetention.value().getStringRep()); } } + if (params.paramAsBoolean(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false)) { + Tuple effectiveRetention = getEffectiveDataRetentionWithSource(globalRetention); + if (effectiveRetention.v1() != null) { + builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveRetention.v1().getStringRep()); + builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveRetention.v2().displayName()); + } + } + if (downsampling != null) { builder.field(DOWNSAMPLING_FIELD.getPreferredName()); downsampling.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index 18a99f984707f..74627e27032b4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -213,14 +213,18 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null); + return toXContent(builder, params, null, null); } /** * Converts the template to XContent and passes the RolloverConditions, when provided, to the lifecycle. */ - public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration) - throws IOException { + public XContentBuilder toXContent( + XContentBuilder builder, + Params params, + @Nullable RolloverConfiguration rolloverConfiguration, + @Nullable DataStreamGlobalRetention globalRetention + ) throws IOException { builder.startObject(); if (this.settings != null) { builder.startObject(SETTINGS.getPreferredName()); @@ -250,7 +254,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla } if (this.lifecycle != null) { builder.field(LIFECYCLE.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration); + lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); } builder.endObject(); return builder; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java index 3b24f90b9d854..2af4bf5016ad2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java @@ -8,15 +8,34 @@ package org.elasticsearch.action.admin.indices.template.get; +import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComponentTemplateTests; +import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSerializationTests; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; -import java.util.Collections; +import java.io.IOException; import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomAliases; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomMappings; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + public class GetComponentTemplateResponseTests extends AbstractWireSerializingTestCase { @Override protected Writeable.Reader instanceReader() { @@ -25,18 +44,84 @@ protected Writeable.Reader instanceReader() @Override protected GetComponentTemplateAction.Response createTestInstance() { + return new GetComponentTemplateAction.Response( + randomBoolean() ? Map.of() : randomTemplates(), + RolloverConfigurationTests.randomRolloverConditions(), + DataStreamGlobalRetentionSerializationTests.randomGlobalRetention() + ); + } + + @Override + protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) { + var templates = instance.getComponentTemplates(); + var rolloverConditions = instance.getRolloverConfiguration(); + var globalRetention = instance.getGlobalRetention(); + switch (randomInt(2)) { + case 0 -> templates = templates == null ? randomTemplates() : null; + case 1 -> rolloverConditions = randomValueOtherThan(rolloverConditions, RolloverConfigurationTests::randomRolloverConditions); + case 2 -> globalRetention = randomValueOtherThan( + globalRetention, + DataStreamGlobalRetentionSerializationTests::randomGlobalRetention + ); + } + return new GetComponentTemplateAction.Response(templates, rolloverConditions, globalRetention); + } + + public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException { + Settings settings = null; + CompressedXContent mappings = null; + Map aliases = null; + DataStreamLifecycle lifecycle = new DataStreamLifecycle(); if (randomBoolean()) { - return new GetComponentTemplateAction.Response(Collections.emptyMap()); + settings = randomSettings(); } - Map templates = new HashMap<>(); - for (int i = 0; i < randomIntBetween(1, 4); i++) { - templates.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance()); + if (randomBoolean()) { + mappings = randomMappings(); + } + if (randomBoolean()) { + aliases = randomAliases(); + } + + var template = new ComponentTemplate( + new Template(settings, mappings, aliases, lifecycle), + randomBoolean() ? null : randomNonNegativeLong(), + null, + false + ); + var globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); + var rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); + var response = new GetComponentTemplateAction.Response( + Map.of(randomAlphaOfLength(10), template), + rolloverConfiguration, + globalRetention + ); + + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContent(builder, EMPTY_PARAMS); + String serialized = Strings.toString(builder); + assertThat(serialized, containsString("rollover")); + for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) + .getConditions() + .keySet()) { + assertThat(serialized, containsString(label)); + } + // We check that even if there was no retention provided by the user, the global retention applies + assertThat(serialized, not(containsString("data_retention"))); + assertThat(serialized, containsString("effective_retention")); } - return new GetComponentTemplateAction.Response(templates); } @Override - protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) { - return randomValueOtherThan(instance, this::createTestInstance); + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(IndicesModule.getNamedWriteables()); + } + + private static Map randomTemplates() { + Map templates = new HashMap<>(); + for (int i = 0; i < randomIntBetween(1, 4); i++) { + templates.put(randomAlphaOfLength(4), ComponentTemplateTests.randomInstance()); + } + return templates; } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java index 8b9ef91923839..7efa624b49148 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java @@ -32,6 +32,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class ComponentTemplateTests extends SimpleDiffableSerializationTestCase { @Override @@ -112,7 +113,7 @@ public static Map randomAliases() { return Collections.singletonMap(aliasName, aliasMeta); } - private static CompressedXContent randomMappings() { + public static CompressedXContent randomMappings() { try { return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}"); } catch (IOException e) { @@ -121,7 +122,7 @@ private static CompressedXContent randomMappings() { } } - private static Settings randomSettings() { + public static Settings randomSettings() { return indexSettings(randomIntBetween(1, 10), randomIntBetween(0, 5)).put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) @@ -265,7 +266,7 @@ public void testMappingsEquals() throws IOException { } } - public void testXContentSerializationWithRollover() throws IOException { + public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException { Settings settings = null; CompressedXContent mappings = null; Map aliases = null; @@ -278,7 +279,7 @@ public void testXContentSerializationWithRollover() throws IOException { if (randomBoolean()) { aliases = randomAliases(); } - DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build(); + DataStreamLifecycle lifecycle = new DataStreamLifecycle(); ComponentTemplate template = new ComponentTemplate( new Template(settings, mappings, aliases, lifecycle), randomNonNegativeLong(), @@ -288,14 +289,19 @@ public void testXContentSerializationWithRollover() throws IOException { try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { builder.humanReadable(true); RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); - template.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration); + DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); + ToXContent.Params withEffectiveRetention = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS); + template.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); assertThat(serialized, containsString("rollover")); - for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention()) + for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) .getConditions() .keySet()) { assertThat(serialized, containsString(label)); } + // We check that even if there was no retention provided by the user, the global retention applies + assertThat(serialized, not(containsString("data_retention"))); + assertThat(serialized, containsString("effective_retention")); } } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index fe678ec23afad..6485634f879ba 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -30,6 +30,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.TIMESTAMP_FIELD_NAME; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class ComposableIndexTemplateTests extends SimpleDiffableSerializationTestCase { @Override @@ -109,10 +110,6 @@ private static Map randomAliases() { return Collections.singletonMap(aliasName, aliasMeta); } - private static DataStreamLifecycle randomLifecycle() { - return DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build(); - } - private static CompressedXContent randomMappings(ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate) { try { if (dataStreamTemplate != null) { @@ -212,7 +209,7 @@ public void testComponentTemplatesEquals() { assertThat(ComposableIndexTemplate.componentTemplatesEquals(List.of(), List.of(randomAlphaOfLength(5))), equalTo(false)); } - public void testXContentSerializationWithRollover() throws IOException { + public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException { Settings settings = null; CompressedXContent mappings = null; Map aliases = null; @@ -226,7 +223,8 @@ public void testXContentSerializationWithRollover() throws IOException { if (randomBoolean()) { aliases = randomAliases(); } - DataStreamLifecycle lifecycle = randomLifecycle(); + // We use the empty lifecycle so the global retention can be in effect + DataStreamLifecycle lifecycle = new DataStreamLifecycle(); Template template = new Template(settings, mappings, aliases, lifecycle); ComposableIndexTemplate.builder() .indexPatterns(List.of(randomAlphaOfLength(4))) @@ -240,14 +238,19 @@ public void testXContentSerializationWithRollover() throws IOException { try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { builder.humanReadable(true); RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); - template.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration); + DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); + ToXContent.Params withEffectiveRetention = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS); + template.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); assertThat(serialized, containsString("rollover")); - for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention()) + for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) .getConditions() .keySet()) { assertThat(serialized, containsString(label)); } + // We check that even if there was no retention provided by the user, the global retention applies + assertThat(serialized, not(containsString("data_retention"))); + assertThat(serialized, containsString("effective_retention")); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSerializationTests.java index 491ba868dfd9b..5cd104f1f59b5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSerializationTests.java @@ -75,9 +75,10 @@ protected ClusterState.Custom mutateInstance(ClusterState.Custom instance) { } public static DataStreamGlobalRetention randomGlobalRetention() { + boolean withDefault = randomBoolean(); return new DataStreamGlobalRetention( - randomBoolean() ? null : TimeValue.timeValueDays(randomIntBetween(1, 1000)), - randomBoolean() ? null : TimeValue.timeValueDays(randomIntBetween(1000, 2000)) + withDefault == false ? null : TimeValue.timeValueDays(randomIntBetween(1, 1000)), + withDefault && randomBoolean() ? null : TimeValue.timeValueDays(randomIntBetween(1000, 2000)) ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index b266addc37407..d389131e6b294 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -39,6 +39,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.MAX_GLOBAL_RETENTION; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; public class DataStreamLifecycleTests extends AbstractXContentSerializingTestCase { @@ -106,13 +107,14 @@ protected DataStreamLifecycle doParseInstance(XContentParser parser) throws IOEx return DataStreamLifecycle.fromXContent(parser); } - public void testXContentSerializationWithRollover() throws IOException { + public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException { DataStreamLifecycle lifecycle = createTestInstance(); try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { builder.humanReadable(true); RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); - lifecycle.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention); + ToXContent.Params withEffectiveRetention = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS); + lifecycle.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); assertThat(serialized, containsString("rollover")); for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) @@ -124,6 +126,13 @@ public void testXContentSerializationWithRollover() throws IOException { if (rolloverConfiguration.getAutomaticConditions().isEmpty() == false) { assertThat(serialized, containsString("[automatic]")); } + // We check that even if there was no retention provided by the user, the global retention applies + if (lifecycle.getDataRetention() == null) { + assertThat(serialized, not(containsString("data_retention"))); + } else { + assertThat(serialized, containsString("data_retention")); + } + assertThat(serialized, containsString("effective_retention")); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 3e758df17c432..a1a523ddb584d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -1660,7 +1660,7 @@ private DataStream createDataStream( return newInstance(dataStreamName, backingIndices, backingIndicesCount, null, false, lifecycle); } - public void testXContentSerializationWithRollover() throws IOException { + public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); List indices = randomIndexInstances(); long generation = indices.size() + ESTestCase.randomLongBetween(1, 128); @@ -1675,7 +1675,7 @@ public void testXContentSerializationWithRollover() throws IOException { failureIndices = randomNonEmptyIndexInstances(); } - DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build(); + DataStreamLifecycle lifecycle = new DataStreamLifecycle(); DataStream dataStream = new DataStream( dataStreamName, indices, @@ -1698,7 +1698,9 @@ public void testXContentSerializationWithRollover() throws IOException { builder.humanReadable(true); RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionSerializationTests.randomGlobalRetention(); - dataStream.toXContent(builder, ToXContent.EMPTY_PARAMS, rolloverConfiguration, globalRetention); + + ToXContent.Params withEffectiveRetention = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS); + dataStream.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention); String serialized = Strings.toString(builder); assertThat(serialized, containsString("rollover")); for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention)) @@ -1706,6 +1708,9 @@ public void testXContentSerializationWithRollover() throws IOException { .keySet()) { assertThat(serialized, containsString(label)); } + // We check that even if there was no retention provided by the user, the global retention applies + assertThat(serialized, not(containsString("data_retention"))); + assertThat(serialized, containsString("effective_retention")); } }