[DSL Global Retention] Use data stream global retention metadata

docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc

@@ -130,14 +130,18 @@ The response will look like the following:
       "name": "my-data-stream-1",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     },
     {
       "name": "my-data-stream-2",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   ]

docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc

@@ -74,7 +74,9 @@ The response will look like:
       "generation_time": "6.84s",                       <9>
       "lifecycle": {
         "enabled": true,
-        "data_retention": "30d"                         <10>
+        "data_retention": "30d",
+        "effective_retention": "30d"                    <10>
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   }

docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc

@@ -93,19 +93,24 @@ The result will look like this:
 {
   "data_streams": [
     {
-      "name": "my-data-stream",<1>
+      "name": "my-data-stream",                                   <1>
       "lifecycle": {
-        "enabled": true,       <2>
-        "data_retention": "7d" <3>
+        "enabled": true,                                          <2>
+        "data_retention": "7d",                                   <3>
+        "effective_retention": "7d",                              <4>
+        "retention_determined_by": "data_stream_configuration"    <5>
       }
     }
   ]
 }
 --------------------------------------------------
 <1> The name of your data stream.
 <2> Shows if the data stream lifecycle is enabled for this data stream.
-<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will
+<3> The desired retention period of the data indexed in this data stream, this means that if there are no other limitation
+the data for this data stream will be preserved for at least 7 days.
+<4> The effective retention, this means that the data in this data stream will
 be kept at least for 7 days. After that {es} can delete it at its own discretion.
+<5> The configuration that determined the effective retention.
 
 If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the
 <<data-streams-explain-lifecycle,explain data stream lifecycle API>>:
@@ -128,7 +133,9 @@ The result will look like this:
       "time_since_index_creation": "1.6m",                  <3>
       "lifecycle": {                                        <4>
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   }

docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc

@@ -200,10 +200,12 @@ PUT _index_template/dsl-data-stream-template
   "template": {
     "settings": {
       "index.lifecycle.name": "pre-dsl-ilm-policy",
-      "index.lifecycle.prefer_ilm": false             <1>
+      "index.lifecycle.prefer_ilm": false                       <1>
     },
-    "lifecycle": {
-      "data_retention": "7d"                          <2>
+    "lifecycle": {                                              <2>
+      "data_retention": "7d",                                   <3>
+      "effective_retention": "7d",                              <4>
+      "retention_determined_by": "data_stream_configuration"    <5>
     }
   }
 }
@@ -215,6 +217,11 @@ PUT _index_template/dsl-data-stream-template
 precedence over data stream lifecycle.
 <2> We're configuring the data stream lifecycle so _new_ data streams will be 
 managed by data stream lifecycle.
+<3> The desired retention, meaning that this data stream should keep the data for at least 7 days,
+if this retention is possible.
+<4> The effective retention is the least amount of time data will be kept by {es}.
+<5> This field displays which configuration determined the effective retention. In this case
+it was the desired retention defined by the user.
 
 We've now made sure that new data streams will be managed by data stream lifecycle.
 
@@ -268,7 +275,9 @@ GET _data_stream/dsl-data-stream
       "template": "dsl-data-stream-template",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       },
       "ilm_policy": "pre-dsl-ilm-policy",
       "next_generation_managed_by": "Data stream lifecycle",         <3>
@@ -346,7 +355,9 @@ GET _data_stream/dsl-data-stream
       "template": "dsl-data-stream-template",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       },
       "ilm_policy": "pre-dsl-ilm-policy",
       "next_generation_managed_by": "Data stream lifecycle",

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -199,7 +200,8 @@ static GetDataStreamAction.Response innerOperation(
         }
         return new GetDataStreamAction.Response(
             dataStreamInfos,
-            request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null
+            request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
+            DataStreamGlobalRetention.getFromClusterState(state)
         );
     }
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.datastreams.lifecycle.action;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -16,6 +17,7 @@
 import org.elasticsearch.action.datastreams.lifecycle.ExplainIndexDataStreamLifecycle;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -136,23 +138,33 @@ public Request indicesOptions(IndicesOptions indicesOptions) {
     }
 
     /**
-     * Class representing the response for the explain of the data stream lifecycle action for one or more indices.
+     * Class representing the response for the 'explain' of the data stream lifecycle action for one or more indices.
      */
     public static class Response extends ActionResponse implements ChunkedToXContentObject {
         public static final ParseField INDICES_FIELD = new ParseField("indices");
-        private List<ExplainIndexDataStreamLifecycle> indices;
+        private final List<ExplainIndexDataStreamLifecycle> indices;
         @Nullable
         private final RolloverConfiguration rolloverConfiguration;
+        @Nullable
+        private final DataStreamGlobalRetention globalRetention;
 
-        public Response(List<ExplainIndexDataStreamLifecycle> indices, @Nullable RolloverConfiguration rolloverConfiguration) {
+        public Response(
+            List<ExplainIndexDataStreamLifecycle> indices,
+            @Nullable RolloverConfiguration rolloverConfiguration,
+            @Nullable DataStreamGlobalRetention globalRetention
+        ) {
             this.indices = indices;
             this.rolloverConfiguration = rolloverConfiguration;
+            this.globalRetention = globalRetention;
         }
 
         public Response(StreamInput in) throws IOException {
             super(in);
             this.indices = in.readCollectionAsList(ExplainIndexDataStreamLifecycle::new);
             this.rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new);
+            this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)
+                ? in.readOptionalWriteable(DataStreamGlobalRetention::read)
+                : null;
         }
 
         public List<ExplainIndexDataStreamLifecycle> getIndices() {
@@ -163,10 +175,17 @@ public RolloverConfiguration getRolloverConfiguration() {
             return rolloverConfiguration;
         }
 
+        public DataStreamGlobalRetention getGlobalRetention() {
+            return globalRetention;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(indices);
             out.writeOptionalWriteable(rolloverConfiguration);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
+                out.writeOptionalWriteable(globalRetention);
+            }
         }
 
         @Override
@@ -178,12 +197,14 @@ public boolean equals(Object o) {
                 return false;
             }
             Response response = (Response) o;
-            return Objects.equals(indices, response.indices) && Objects.equals(rolloverConfiguration, response.rolloverConfiguration);
+            return Objects.equals(indices, response.indices)
+                && Objects.equals(rolloverConfiguration, response.rolloverConfiguration)
+                && Objects.equals(globalRetention, response.globalRetention);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(indices, rolloverConfiguration);
+            return Objects.hash(indices, rolloverConfiguration, globalRetention);
         }
 
         @Override
@@ -194,7 +215,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                 return builder;
             }), Iterators.map(indices.iterator(), explainIndexDataLifecycle -> (builder, params) -> {
                 builder.field(explainIndexDataLifecycle.getIndex());
-                explainIndexDataLifecycle.toXContent(builder, params, rolloverConfiguration);
+                explainIndexDataLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
                 return builder;
             }), Iterators.single((builder, params) -> {
                 builder.endObject();

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java

@@ -7,13 +7,15 @@
  */
 package org.elasticsearch.datastreams.lifecycle.action;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -157,19 +159,24 @@ public void writeTo(StreamOutput out) throws IOException {
 
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                return toXContent(builder, params, null);
+                return toXContent(builder, params, null, null);
             }
 
             /**
-             * Converts the response to XContent and passes the RolloverConditions, when provided, to the data stream lifecycle.
+             * Converts the response to XContent and passes the RolloverConditions and the global retention, when provided,
+             * to the data stream lifecycle.
              */
-            public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration)
-                throws IOException {
+            public XContentBuilder toXContent(
+                XContentBuilder builder,
+                Params params,
+                @Nullable RolloverConfiguration rolloverConfiguration,
+                @Nullable DataStreamGlobalRetention globalRetention
+            ) throws IOException {
                 builder.startObject();
                 builder.field(NAME_FIELD.getPreferredName(), dataStreamName);
                 if (lifecycle != null) {
                     builder.field(LIFECYCLE_FIELD.getPreferredName());
-                    lifecycle.toXContent(builder, params, rolloverConfiguration);
+                    lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);
                 }
                 builder.endObject();
                 return builder;
@@ -179,18 +186,31 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
         private final List<DataStreamLifecycle> dataStreamLifecycles;
         @Nullable
         private final RolloverConfiguration rolloverConfiguration;
+        @Nullable
+        private final DataStreamGlobalRetention globalRetention;
 
         public Response(List<DataStreamLifecycle> dataStreamLifecycles) {
-            this(dataStreamLifecycles, null);
+            this(dataStreamLifecycles, null, null);
         }
 
-        public Response(List<DataStreamLifecycle> dataStreamLifecycles, @Nullable RolloverConfiguration rolloverConfiguration) {
+        public Response(
+            List<DataStreamLifecycle> dataStreamLifecycles,
+            @Nullable RolloverConfiguration rolloverConfiguration,
+            @Nullable DataStreamGlobalRetention globalRetention
+        ) {
             this.dataStreamLifecycles = dataStreamLifecycles;
             this.rolloverConfiguration = rolloverConfiguration;
+            this.globalRetention = globalRetention;
         }
 
         public Response(StreamInput in) throws IOException {
-            this(in.readCollectionAsList(Response.DataStreamLifecycle::new), in.readOptionalWriteable(RolloverConfiguration::new));
+            this(
+                in.readCollectionAsList(Response.DataStreamLifecycle::new),
+                in.readOptionalWriteable(RolloverConfiguration::new),
+                in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)
+                    ? in.readOptionalWriteable(DataStreamGlobalRetention::read)
+                    : null
+            );
         }
 
         public List<DataStreamLifecycle> getDataStreamLifecycles() {
@@ -206,6 +226,9 @@ public RolloverConfiguration getRolloverConfiguration() {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(dataStreamLifecycles);
             out.writeOptionalWriteable(rolloverConfiguration);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
+                out.writeOptionalWriteable(globalRetention);
+            }
         }
 
         @Override
@@ -217,7 +240,12 @@ public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
             }),
                 Iterators.map(
                     dataStreamLifecycles.iterator(),
-                    dataStreamLifecycle -> (builder, params) -> dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration)
+                    dataStreamLifecycle -> (builder, params) -> dataStreamLifecycle.toXContent(
+                        builder,
+                        params,
+                        rolloverConfiguration,
+                        globalRetention
+                    )
                 ),
                 Iterators.single((builder, params) -> {
                     builder.endArray();
@@ -233,12 +261,13 @@ public boolean equals(Object o) {
             if (o == null || getClass() != o.getClass()) return false;
             Response response = (Response) o;
             return dataStreamLifecycles.equals(response.dataStreamLifecycles)
-                && Objects.equals(rolloverConfiguration, response.rolloverConfiguration);
+                && Objects.equals(rolloverConfiguration, response.rolloverConfiguration)
+                && Objects.equals(globalRetention, response.globalRetention);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(dataStreamLifecycles, rolloverConfiguration);
+            return Objects.hash(dataStreamLifecycles, rolloverConfiguration, globalRetention);
         }
     }
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -111,7 +112,8 @@ protected void masterOperation(
         listener.onResponse(
             new ExplainDataStreamLifecycleAction.Response(
                 explainIndices,
-                request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null
+                request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
+                DataStreamGlobalRetention.getFromClusterState(state)
             )
         );
     }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -89,7 +90,8 @@ protected void masterOperation(
                     )
                     .sorted(Comparator.comparing(GetDataStreamLifecycleAction.Response.DataStreamLifecycle::dataStreamName))
                     .toList(),
-                request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null
+                request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
+                DataStreamGlobalRetention.getFromClusterState(state)
             )
         );
     }