TSDB: Make time boundaries settings required in tsdb indices (#81146)

Make `index.time_series.start_time` and `index.time_series.end_time` settings as required settings in tsdb indices.

It will change many tests, among which a time_series index is created, and it will add the `index.time_series.start_time` and `index.time_series.end_time` settings

Author: weizijun

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedFunction;
@@ -307,6 +308,8 @@ private int createTimeSeriesModeIndex(int replicas) throws IOException {
             mappingsAndSettings.field("number_of_replicas", replicas);
             mappingsAndSettings.field("mode", "time_series");
             mappingsAndSettings.field("routing_path", "dim");
+            mappingsAndSettings.field("time_series.start_time", 1L);
+            mappingsAndSettings.field("time_series.end_time", DateUtils.MAX_MILLIS_BEFORE_9999 - 1);
             mappingsAndSettings.endObject();
         }
         {

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_tsdb.yml

@@ -2,8 +2,8 @@
 setup:
   # Create a local tsdb index with a tsid the doesn't overlap with the remote cluster.
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       indices.create:
@@ -13,6 +13,9 @@ setup:
             index:
               mode: time_series
               routing_path: [metricset, k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
               number_of_replicas: 0
               number_of_shards: 2
           mappings:

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -156,8 +156,8 @@
 ---
 tsdb:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       indices.create:
@@ -167,6 +167,9 @@ tsdb:
             index:
               mode: time_series
               routing_path: [metricset, k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
               number_of_replicas: 0
               number_of_shards: 2
           mappings:

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -322,6 +323,8 @@ private void createTsdbIndex() throws IOException {
         indexSpec.startObject("settings").startObject("index");
         indexSpec.field("mode", "time_series");
         indexSpec.array("routing_path", new String[] { "dim" });
+        indexSpec.field("time_series.start_time", 1L);
+        indexSpec.field("time_series.end_time", DateUtils.MAX_MILLIS_BEFORE_9999 - 1);
         indexSpec.endObject().endObject();
         createIndex.setJsonEntity(Strings.toString(indexSpec.endObject()));
         client().performRequest(createIndex);

qa/smoke-test-multinode/src/test/resources/rest-api-spec/test/smoke_test_multinode/20_tsdb_consistency.yml

@@ -3,8 +3,8 @@
 
 setup:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       indices.create:
@@ -14,6 +14,9 @@ setup:
               index:
                 mode: time_series
                 routing_path: [metricset, k8s.pod.uid]
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 number_of_shards: 10
                 number_of_replicas: 1
             mappings:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

@@ -1,7 +1,7 @@
 enable:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       indices.create:
@@ -11,6 +11,9 @@ enable:
               index:
                 mode: time_series
                 routing_path: [metricset, k8s.pod.uid]
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 number_of_replicas: 0
                 number_of_shards: 2
             mappings:
@@ -41,8 +44,8 @@ enable:
 ---
 no sort field:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.mode=time_series\] is incompatible with \[index.sort.field\]/
@@ -52,13 +55,17 @@ no sort field:
             settings:
               index:
                 mode: time_series
+                routing_path: foo
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 sort.field: ['a']
 
 ---
 no sort order:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.mode=time_series\] is incompatible with \[index.sort.order\]/
@@ -68,13 +75,17 @@ no sort order:
             settings:
               index:
                 mode: time_series
+                routing_path: foo
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 sort.order: ['DESC']
 
 ---
 no sort mode:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.mode=time_series\] is incompatible with \[index.sort.mode\]/
@@ -84,13 +95,17 @@ no sort mode:
             settings:
               index:
                 mode: time_series
+                routing_path: foo
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 sort.mode: ['MIN']
 
 ---
 no sort missing:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.mode=time_series\] is incompatible with \[index.sort.missing\]/
@@ -100,13 +115,17 @@ no sort missing:
             settings:
               index:
                 mode: time_series
+                routing_path: foo
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 sort.missing: ['_last']
 
 ---
 no partitioning:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.mode=time_series\] is incompatible with \[index.routing_partition_size\]/
@@ -116,14 +135,18 @@ no partitioning:
             settings:
               index:
                 mode: time_series
+                routing_path: foo
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
                 shards: 5
                 routing_partition_size: 2
 
 ---
 routing_path required:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0 to be backported to 7.16.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.mode=time_series\] requires \[index.routing_path\]/
@@ -133,13 +156,16 @@ routing_path required:
             settings:
               index:
                 mode: time_series
-                shards: 5
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
+                number_of_shards: 5
 
 ---
 routing_path is not allowed in standard mode:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0 to be backported to 7.16.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /\[index.routing_path\] requires \[index.mode=time_series\]/
@@ -153,8 +179,8 @@ routing_path is not allowed in standard mode:
 ---
 routing required:
   - skip:
-      version: " - 7.99.99"
-      reason: introduced in 8.0.0 to be backported to 7.16.0
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
 
   - do:
       catch: /routing is forbidden on CRUD operations that target indices in \[index.mode=time_series\]/
@@ -165,6 +191,9 @@ routing required:
               index:
                 mode: time_series
                 routing_path: foo
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
             mappings:
               _routing:
                 required: true