[Mapper] Add ignore_missing option to timestamp

Related to #9049.

By default, the default value for `timestamp` is `now` which means the date the document was processed by the indexing chain.

You can now reject documents which not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):

```js
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "ignore_missing" : false
        }
    }
}
```

When you update the cluster to 1.5 or master, this index created with 1.4 we automatically migrate an index created with 1.4 to the 1.5 syntax.

Let say you have defined this in elasticsearch 1.4.x:

```js
DELETE test
PUT test
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}
PUT test/type/_mapping
{
  "type" : {
      "_timestamp" : {
          "enabled" : true,
          "default" : null
      }
  }
}
```

After migration, the mapping become:

```js
{
   "test": {
      "mappings": {
         "type": {
            "_timestamp": {
               "enabled": true,
               "store": false,
               "ignore_missing": false
            },
            "properties": {}
         }
      }
   }
}
```

Closes #8882.
(cherry picked from commit fb10346)

Author: dadoonet

docs/reference/mapping/fields/timestamp-field.asciidoc

@@ -91,22 +91,20 @@ within the index request or in the `_source` document.
 
 By default, the default value is `now` which means the date the document was processed by the indexing chain.
 
-You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
+You can reject documents which do not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):
 
 [source,js]
 --------------------------------------------------
 {
     "tweet" : {
         "_timestamp" : {
             "enabled" : true,
-            "default" : null
+            "ignore_missing" : false
         }
     }
 }
 --------------------------------------------------
 
-If you don't provide any timestamp value, indexation will fail.
-
 You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:
 
 [source,js]
@@ -124,3 +122,10 @@ You can also set the default value to any date respecting <<mapping-timestamp-fi
 
 If you don't provide any timestamp value, _timestamp will be set to this default value.
 
+coming[1.5.0]
+
+In elasticsearch 1.4, we allowed setting explicitly `"default":null` which is not possible anymore
+as we added a new `ignore_missing`setting.
+When reading an index created with elasticsearch 1.4 and using this, we automatically update it by
+removing `"default": null` and setting `"ignore_missing": false`
+

src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -21,11 +21,7 @@
 
 import com.google.common.base.Charsets;
 import org.elasticsearch.*;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.RoutingMissingException;
-import org.elasticsearch.action.TimestampParsingException;
-import org.elasticsearch.action.DocumentRequest;
+import org.elasticsearch.action.*;
 import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -659,11 +655,13 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
         if (timestamp == null) {
             String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
             if (mappingMd != null && mappingMd.timestamp() != null) {
+                // If we explicitly ask to reject null timestamp
+                if (mappingMd.timestamp().ignoreMissing() != null && mappingMd.timestamp().ignoreMissing() == false) {
+                    throw new TimestampParsingException("timestamp is required by mapping");
+                }
                 defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
             }
-            if (!Strings.hasText(defaultTimestamp)) {
-                throw new TimestampParsingException("timestamp is required by mapping");
-            }
+
             if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
                 timestamp = Long.toString(System.currentTimeMillis());
             } else {

src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java

@@ -177,7 +177,7 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
 
 
         public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
-                TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
+                TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null);
 
         private final boolean enabled;
 
@@ -191,7 +191,9 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
 
         private final String defaultTimestamp;
 
-        public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
+        private final Boolean ignoreMissing;
+
+        public Timestamp(boolean enabled, String path, String format, String defaultTimestamp, Boolean ignoreMissing) {
             this.enabled = enabled;
             this.path = path;
             if (path == null) {
@@ -202,6 +204,7 @@ public Timestamp(boolean enabled, String path, String format, String defaultTime
             this.format = format;
             this.dateTimeFormatter = Joda.forPattern(format);
             this.defaultTimestamp = defaultTimestamp;
+            this.ignoreMissing = ignoreMissing;
         }
 
         public boolean enabled() {
@@ -232,6 +235,10 @@ public boolean hasDefaultTimestamp() {
             return this.defaultTimestamp != null;
         }
 
+        public Boolean ignoreMissing() {
+            return ignoreMissing;
+        }
+
         public FormatDateTimeFormatter dateTimeFormatter() {
             return this.dateTimeFormatter;
         }
@@ -247,6 +254,7 @@ public boolean equals(Object o) {
             if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
             if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
             if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
+            if (ignoreMissing != null ? !ignoreMissing.equals(timestamp.ignoreMissing) : timestamp.ignoreMissing != null) return false;
             if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;
 
             return true;
@@ -260,6 +268,7 @@ public int hashCode() {
             result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
             result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
             result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
+            result = 31 * result + (ignoreMissing != null ? ignoreMissing.hashCode() : 0);
             return result;
         }
     }
@@ -278,7 +287,9 @@ public MappingMetaData(DocumentMapper docMapper) {
         this.source = docMapper.mappingSource();
         this.id = new Id(docMapper.idFieldMapper().path());
         this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
-        this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
+        this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(),
+                docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp(),
+                docMapper.timestampFieldMapper().ignoreMissing());
         this.hasParentField = docMapper.parentFieldMapper().active();
     }
 
@@ -344,6 +355,7 @@ private void initMappers(Map<String, Object> withoutType) {
             String path = null;
             String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
             String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
+            Boolean ignoreMissing = null;
             Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
             for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
                 String fieldName = Strings.toUnderscoreCase(entry.getKey());
@@ -356,9 +368,11 @@ private void initMappers(Map<String, Object> withoutType) {
                     format = fieldNode.toString();
                 } else if (fieldName.equals("default") && fieldNode != null) {
                     defaultTimestamp = fieldNode.toString();
+                } else if (fieldName.equals("ignore_missing")) {
+                    ignoreMissing = nodeBooleanValue(fieldNode);
                 }
             }
-            this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
+            this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
         } else {
             this.timestamp = Timestamp.EMPTY;
         }
@@ -554,6 +568,10 @@ public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws I
                 out.writeBoolean(false);
             }
         }
+        // TODO Remove the test in elasticsearch 2.0.0
+        if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
+            out.writeOptionalBoolean(mappingMd.timestamp().ignoreMissing());
+        }
         out.writeBoolean(mappingMd.hasParentField());
     }
 
@@ -591,8 +609,19 @@ public static MappingMetaData readFrom(StreamInput in) throws IOException {
         // routing
         Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
         // timestamp
-        Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(),
-                in.getVersion().onOrAfter(Version.V_1_4_0_Beta1) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
+
+        boolean enabled = in.readBoolean();
+        String path = in.readOptionalString();
+        String format = in.readString();
+        String defaultTimestamp = in.getVersion().onOrAfter(Version.V_1_4_0_Beta1) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
+        Boolean ignoreMissing = null;
+
+        // TODO Remove the test in elasticsearch 2.0.0
+        if (in.getVersion().onOrAfter(Version.V_1_5_0)) {
+            ignoreMissing = in.readOptionalBoolean();
+        }
+
+        final Timestamp timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
         final boolean hasParentField = in.readBoolean();
         return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
     }

src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java

@@ -22,6 +22,8 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.NumericDocValuesField;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -80,6 +82,7 @@ public static class Builder extends NumberFieldMapper.Builder<Builder, Timestamp
         private String path = Defaults.PATH;
         private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
         private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
+        private Boolean ignoreMissing = null;
 
         public Builder() {
             super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@@ -105,6 +108,11 @@ public Builder defaultTimestamp(String defaultTimestamp) {
             return builder;
         }
 
+        public Builder ignoreMissing(boolean ignoreMissing) {
+            this.ignoreMissing = ignoreMissing;
+            return builder;
+        }
+
         @Override
         public TimestampFieldMapper build(BuilderContext context) {
             boolean roundCeil = Defaults.ROUND_CEIL;
@@ -113,6 +121,7 @@ public TimestampFieldMapper build(BuilderContext context) {
                 roundCeil =  settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
             }
             return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
+                    ignoreMissing,
                     ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
         }
     }
@@ -122,6 +131,8 @@ public static class TypeParser implements Mapper.TypeParser {
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             TimestampFieldMapper.Builder builder = timestamp();
             parseField(builder, builder.name, node, parserContext);
+            boolean defaultSet = false;
+            Boolean ignoreMissing = null;
             for (Map.Entry<String, Object> entry : node.entrySet()) {
                 String fieldName = Strings.toUnderscoreCase(entry.getKey());
                 Object fieldNode = entry.getValue();
@@ -134,8 +145,31 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     builder.dateTimeFormatter(parseDateTimeFormatter(fieldNode.toString()));
                 } else if (fieldName.equals("default")) {
                     builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
+                    if (fieldNode == null) {
+                        if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_4_0_Beta1) &&
+                                parserContext.indexVersionCreated().before(Version.V_1_5_0)) {
+                            // We are reading an index created in 1.4 with feature #7036
+                            // `default: null` was explicitly set. We need to change this index to
+                            // `ignore_missing: false`
+                            builder.ignoreMissing(false);
+                        } else {
+                            throw new TimestampParsingException("default timestamp can not be set to null");
+                        }
+                    } else {
+                        builder.defaultTimestamp(fieldNode.toString());
+                        defaultSet = true;
+                    }
+                } else if (fieldName.equals("ignore_missing")) {
+                    ignoreMissing = nodeBooleanValue(fieldNode);
+                    builder.ignoreMissing(ignoreMissing);
                 }
             }
+
+            // We can not accept a default value and rejecting null values at the same time
+            if (defaultSet && (ignoreMissing != null && ignoreMissing == false)) {
+                throw new TimestampParsingException("default timestamp can not be set with ignore_missing set to false");
+            }
+
             return builder;
         }
     }
@@ -145,14 +179,16 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     private final String path;
     private final String defaultTimestamp;
+    private final Boolean ignoreMissing;
 
     public TimestampFieldMapper(Settings indexSettings) {
         this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
-                Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
+                Defaults.ROUND_CEIL, null, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
     }
 
     protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
                                    FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
+                                   Boolean ignoreMissing,
                                    Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
                                    DocValuesFormatProvider docValuesProvider, Loading normsLoading,
                                    @Nullable Settings fieldDataSettings, Settings indexSettings) {
@@ -164,6 +200,7 @@ protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAt
         this.enabledState = enabledState;
         this.path = path;
         this.defaultTimestamp = defaultTimestamp;
+        this.ignoreMissing = ignoreMissing;
     }
 
     @Override
@@ -183,6 +220,10 @@ public String defaultTimestamp() {
         return this.defaultTimestamp;
     }
 
+    public Boolean ignoreMissing() {
+        return this.ignoreMissing;
+    }
+
     public FormatDateTimeFormatter dateTimeFormatter() {
         return this.dateTimeFormatter;
     }
@@ -269,6 +310,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
             builder.field("default", defaultTimestamp);
         }
+        if (includeDefaults || ignoreMissing != null) {
+            builder.field("ignore_missing", ignoreMissing);
+        }
         if (customFieldDataSettings != null) {
             builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
         } else if (includeDefaults) {