TSDB: add index timestamp range check

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml

@@ -168,3 +168,65 @@ routing required:
             mappings:
               _routing:
                 required: true
+---
+set start_time and end_time:
+  - skip:
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [metricset]
+              time_series:
+                start_time: 1632625782000
+                end_time: 1632625792000
+
+  - do:
+      indices.put_settings:
+        index: test_index
+        body:
+          index:
+            time_series:
+              end_time: 1632625793000
+
+  - do:
+      catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/
+      indices.put_settings:
+        index: test_index
+        body:
+          index:
+            time_series:
+              end_time: 1632625792000
+
+  - do:
+      indices.delete:
+        index: test_index
+
+---
+set start_time and end_time without timeseries mode:
+  - skip:
+      version: " - 8.0.99"
+      reason: introduced in 8.1.0
+  - do:
+      catch: /index.time_series.start_time need to be used for time_series mode/
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              time_series:
+                start_time: 1632625782000
+
+  - do:
+      catch: /index.time_series.end_time need to be used for time_series mode/
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              time_series:
+                end_time: 1632625782000

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -184,6 +184,8 @@ private static Set<Setting<?>> builtInIndexSettings() {
         Set<Setting<?>> result = new HashSet<>(ALWAYS_ENABLED_BUILT_IN_INDEX_SETTINGS);
         result.add(IndexSettings.MODE);
         result.add(IndexMetadata.INDEX_ROUTING_PATH);
+        result.add(IndexSettings.TIME_SERIES_START_TIME);
+        result.add(IndexSettings.TIME_SERIES_END_TIME);
         return Set.copyOf(result);
     }
 

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -31,6 +32,7 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
@@ -41,6 +43,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
@@ -1331,6 +1334,21 @@ public static Setting<Long> longSetting(String key, long defaultValue, long minV
         );
     }
 
+    public static Setting<Optional<Instant>> dateSetting(
+        String key,
+        Optional<Instant> defaultValue,
+        Validator<Optional<Instant>> validator,
+        Property... properties
+    ) {
+        return new Setting<Optional<Instant>>(
+            key,
+            defaultValue.map(Instant::toString).orElse(null),
+            s -> Optional.ofNullable(s).map(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER::parse).map(Instant::from),
+            validator,
+            properties
+        );
+    }
+
     public static Setting<String> simpleString(String key, Property... properties) {
         return new Setting<>(key, s -> "", Function.identity(), properties);
     }

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -32,18 +32,23 @@
 
 /**
  * "Mode" that controls which behaviors and settings an index supports.
+ * <p>
+ * For the most part this class concentrates on validating settings and
+ * mappings. Most different behavior is controlled by forcing settings
+ * to be set or not set and by enabling extra fields in the mapping. 
  */
 public enum IndexMode {
     STANDARD {
         @Override
         void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
-            if (false == Objects.equals(
-                IndexMetadata.INDEX_ROUTING_PATH.getDefault(Settings.EMPTY),
-                settings.get(IndexMetadata.INDEX_ROUTING_PATH)
-            )) {
-                throw new IllegalArgumentException(
-                    "[" + IndexMetadata.INDEX_ROUTING_PATH.getKey() + "] requires [" + IndexSettings.MODE.getKey() + "=time_series]"
-                );
+            settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
+            settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
+            settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME);
+        }
+
+        private void settingRequiresTimeSeries(Map<Setting<?>, Object> settings, Setting<?> setting) {
+            if (false == Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) {
+                throw new IllegalArgumentException("[" + setting.getKey() + "] requires [" + IndexSettings.MODE.getKey() + "=time_series]");
             }
         }
 
@@ -67,10 +72,14 @@ void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
                     throw new IllegalArgumentException(error(unsupported));
                 }
             }
-            if (IndexMetadata.INDEX_ROUTING_PATH.getDefault(Settings.EMPTY).equals(settings.get(IndexMetadata.INDEX_ROUTING_PATH))) {
-                throw new IllegalArgumentException(
-                    "[" + IndexSettings.MODE.getKey() + "=time_series] requires [" + IndexMetadata.INDEX_ROUTING_PATH.getKey() + "]"
-                );
+            settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
+            settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
+            settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME);
+        }
+
+        private void settingRequiresTimeSeries(Map<Setting<?>, Object> settings, Setting<?> setting) {
+            if (Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) {
+                throw new IllegalArgumentException("[" + IndexSettings.MODE.getKey() + "=time_series] requires [" + setting.getKey() + "]");
             }
         }
 
@@ -150,7 +159,12 @@ private void validateTimeStampField(Object timestampFieldValue) {
 
     static final List<Setting<?>> VALIDATE_WITH_SETTINGS = List.copyOf(
         Stream.concat(
-            Stream.of(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, IndexMetadata.INDEX_ROUTING_PATH),
+            Stream.of(
+                IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING,
+                IndexMetadata.INDEX_ROUTING_PATH,
+                IndexSettings.TIME_SERIES_START_TIME,
+                IndexSettings.TIME_SERIES_END_TIME
+            ),
             TIME_SERIES_UNSUPPORTED.stream()
         ).collect(toSet())
     );

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -26,10 +26,12 @@
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.node.Node;
 
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -474,6 +476,55 @@ public static boolean isTimeSeriesModeEnabled() {
         return Build.CURRENT.isSnapshot() || (TIME_SERIES_MODE_FEATURE_FLAG_REGISTERED != null && TIME_SERIES_MODE_FEATURE_FLAG_REGISTERED);
     }
 
+    /**
+     * in time series mode, the start time of the index, timestamp must larger than start_time
+     */
+    public static final Setting<Optional<Instant>> TIME_SERIES_START_TIME = Setting.dateSetting(
+        "index.time_series.start_time",
+        Optional.empty(),
+        v -> {},
+        Property.IndexScope,
+        Property.Final
+    );
+
+    /**
+     * in time series mode, the end time of the index, timestamp must smaller than start_time
+     */
+    public static final Setting<Optional<Instant>> TIME_SERIES_END_TIME = Setting.dateSetting(
+        "index.time_series.end_time",
+        Optional.empty(),
+        new Setting.Validator<>() {
+            @Override
+            public void validate(Optional<Instant> value) {}
+
+            @Override
+            public void validate(Optional<Instant> value, Map<Setting<?>, Object> settings) {
+                @SuppressWarnings("unchecked")
+                Optional<Instant> startTime = (Optional<Instant>) settings.get(TIME_SERIES_START_TIME);
+                if (value.isEmpty()) {
+                    if (startTime.isPresent()) {
+                        throw new IllegalArgumentException("index.time_series.start_time must be set with index.time_series.end_time");
+                    }
+                    return;
+                }
+                if (startTime.isEmpty()) {
+                    throw new IllegalArgumentException("index.time_series.start_time must be set with index.time_series.end_time");
+                }
+                if (startTime.get().compareTo(value.get()) < 0) {
+                    throw new IllegalArgumentException("index.time_series.start_time must be larger than index.time_series.end_time");
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                List<Setting<?>> settings = List.of(TIME_SERIES_START_TIME);
+                return settings.iterator();
+            }
+        },
+        Property.IndexScope,
+        Property.Dynamic
+    );
+
     /**
      * The {@link IndexMode "mode"} of the index.
      */
@@ -509,6 +560,14 @@ public Iterator<Setting<?>> settings() {
      * The {@link IndexMode "mode"} of the index.
      */
     private final IndexMode mode;
+    /**
+     * Start time of the time_series index.
+     */
+    private final Instant timeSeriesStartTime;
+    /**
+     * End time of the time_series index.
+     */
+    private volatile Instant timeSeriesEndTime;
 
     // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
     private volatile Settings settings;
@@ -651,6 +710,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         this.indexMetadata = indexMetadata;
         numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null);
         mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD;
+        timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings).orElse(null);
+        timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings).orElse(null);
 
         this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -765,6 +826,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit);
+        scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_END_TIME, this::updateTimeSeriesEndTime);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1278,4 +1340,26 @@ public long getMappingDimensionFieldsLimit() {
     private void setMappingDimensionFieldsLimit(long value) {
         this.mappingDimensionFieldsLimit = value;
     }
+
+    public Instant getTimeSeriesStartTime() {
+        return timeSeriesStartTime;
+    }
+
+    public Instant getTimeSeriesEndTime() {
+        return timeSeriesEndTime;
+    }
+
+    public void updateTimeSeriesEndTime(Optional<Instant> endTime) {
+        if (endTime.isEmpty()) {
+            this.timeSeriesEndTime = null;
+            return;
+        }
+        Instant endInstant = endTime.get();
+        if (this.timeSeriesEndTime.isAfter(endInstant)) {
+            throw new IllegalArgumentException(
+                "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]"
+            );
+        }
+        this.timeSeriesEndTime = endInstant;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;