Skip to content

Commit 85eb0ea

Browse files
dadoonetdadoonet
committed
Generate timestamp when path is null
Index process fails when having `_timestamp` enabled and `path` option is set. It fails with a `TimestampParsingException[failed to parse timestamp [null]]` message. Reproduction: ``` DELETE test PUT test { "mappings": { "test": { "_timestamp" : { "enabled" : "yes", "path" : "post_date" } } } } PUT test/test/1 { "foo": "bar" } ``` You can define a default value for when timestamp is not provided within the index request or in the `_source` document. By default, the default value is `now` which means the date the document was processed by the indexing chain. You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory: ``` { "tweet" : { "_timestamp" : { "enabled" : true, "default" : null } } } ``` If you don't provide any timestamp value, indexation will fail. You can also set the default value to any date respecting timestamp format: ``` { "tweet" : { "_timestamp" : { "enabled" : true, "format" : "YYYY-MM-dd", "default" : "1970-01-01" } } } ``` If you don't provide any timestamp value, indexation will fail. Closes #4718. Closes #7036.
1 parent 6b39aa6 commit 85eb0ea

File tree

6 files changed

+387
-31
lines changed

6 files changed

+387
-31
lines changed

‎docs/reference/mapping/fields/timestamp-field.asciidoc‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
The `_timestamp` field allows to automatically index the timestamp of a
55
document. It can be provided externally via the index request or in the
66
`_source`. If it is not provided externally it will be automatically set
7-
to the date the document was processed by the indexing chain.
7+
to a <<mapping-timestamp-field-default,default date>>.
88

99
[float]
1010
==== enabled
@@ -60,6 +60,7 @@ Note, using `path` without explicit timestamp value provided requires an
6060
additional (though quite fast) parsing phase.
6161

6262
[float]
63+
[[mapping-timestamp-field-format]]
6364
==== format
6465

6566
You can define the <<mapping-date-format,date
@@ -80,3 +81,46 @@ format>> used to parse the provided timestamp value. For example:
8081

8182
Note, the default format is `dateOptionalTime`. The timestamp value will
8283
first be parsed as a number and if it fails the format will be tried.
84+
85+
[float]
86+
[[mapping-timestamp-field-default]]
87+
==== default
88+
89+
You can define a default value for when timestamp is not provided
90+
within the index request or in the `_source` document.
91+
92+
By default, the default value is `now` which means the date the document was processed by the indexing chain.
93+
94+
You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
95+
96+
[source,js]
97+
--------------------------------------------------
98+
{
99+
"tweet" : {
100+
"_timestamp" : {
101+
"enabled" : true,
102+
"default" : null
103+
}
104+
}
105+
}
106+
--------------------------------------------------
107+
108+
If you don't provide any timestamp value, indexation will fail.
109+
110+
You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:
111+
112+
[source,js]
113+
--------------------------------------------------
114+
{
115+
"tweet" : {
116+
"_timestamp" : {
117+
"enabled" : true,
118+
"format" : "YYYY-MM-dd",
119+
"default" : "1970-01-01"
120+
}
121+
}
122+
}
123+
--------------------------------------------------
124+
125+
If you don't provide any timestamp value, indexation will fail.
126+

‎src/main/java/org/elasticsearch/action/index/IndexRequest.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.*;
2424
import org.elasticsearch.action.ActionRequestValidationException;
2525
import org.elasticsearch.action.RoutingMissingException;
26+
import org.elasticsearch.action.TimestampParsingException;
2627
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
2728
import org.elasticsearch.client.Requests;
2829
import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -574,7 +575,9 @@ public void process(MetaData metaData, String aliasOrIndex, @Nullable MappingMet
574575
}
575576
if (parseContext.shouldParseTimestamp()) {
576577
timestamp = parseContext.timestamp();
577-
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
578+
if (timestamp != null) {
579+
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
580+
}
578581
}
579582
} catch (MapperParsingException e) {
580583
throw e;
@@ -613,7 +616,18 @@ public void process(MetaData metaData, String aliasOrIndex, @Nullable MappingMet
613616

614617
// generate timestamp if not provided, we always have one post this stage...
615618
if (timestamp == null) {
616-
timestamp = Long.toString(System.currentTimeMillis());
619+
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
620+
if (mappingMd != null && mappingMd.timestamp() != null) {
621+
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
622+
}
623+
if (!Strings.hasText(defaultTimestamp)) {
624+
throw new TimestampParsingException("timestamp is required by mapping");
625+
}
626+
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
627+
timestamp = Long.toString(System.currentTimeMillis());
628+
} else {
629+
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
630+
}
617631
}
618632
}
619633

‎src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.cluster.metadata;
2121

2222
import org.elasticsearch.ElasticsearchIllegalStateException;
23+
import org.elasticsearch.Version;
2324
import org.elasticsearch.action.TimestampParsingException;
2425
import org.elasticsearch.common.Nullable;
2526
import org.elasticsearch.common.Strings;
@@ -175,7 +176,8 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
175176
}
176177

177178

178-
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT);
179+
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
180+
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
179181

180182
private final boolean enabled;
181183

@@ -187,7 +189,9 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
187189

188190
private final FormatDateTimeFormatter dateTimeFormatter;
189191

190-
public Timestamp(boolean enabled, String path, String format) {
192+
private final String defaultTimestamp;
193+
194+
public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
191195
this.enabled = enabled;
192196
this.path = path;
193197
if (path == null) {
@@ -197,6 +201,7 @@ public Timestamp(boolean enabled, String path, String format) {
197201
}
198202
this.format = format;
199203
this.dateTimeFormatter = Joda.forPattern(format);
204+
this.defaultTimestamp = defaultTimestamp;
200205
}
201206

202207
public boolean enabled() {
@@ -219,6 +224,14 @@ public String format() {
219224
return this.format;
220225
}
221226

227+
public String defaultTimestamp() {
228+
return this.defaultTimestamp;
229+
}
230+
231+
public boolean hasDefaultTimestamp() {
232+
return this.defaultTimestamp != null;
233+
}
234+
222235
public FormatDateTimeFormatter dateTimeFormatter() {
223236
return this.dateTimeFormatter;
224237
}
@@ -233,6 +246,7 @@ public boolean equals(Object o) {
233246
if (enabled != timestamp.enabled) return false;
234247
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
235248
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
249+
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
236250
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;
237251

238252
return true;
@@ -245,6 +259,7 @@ public int hashCode() {
245259
result = 31 * result + (format != null ? format.hashCode() : 0);
246260
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
247261
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
262+
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
248263
return result;
249264
}
250265
}
@@ -263,7 +278,7 @@ public MappingMetaData(DocumentMapper docMapper) {
263278
this.source = docMapper.mappingSource();
264279
this.id = new Id(docMapper.idFieldMapper().path());
265280
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
266-
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
281+
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
267282
this.hasParentField = docMapper.parentFieldMapper().active();
268283
}
269284

@@ -328,6 +343,7 @@ private void initMappers(Map<String, Object> withoutType) {
328343
boolean enabled = false;
329344
String path = null;
330345
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
346+
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
331347
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
332348
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
333349
String fieldName = Strings.toUnderscoreCase(entry.getKey());
@@ -338,9 +354,11 @@ private void initMappers(Map<String, Object> withoutType) {
338354
path = fieldNode.toString();
339355
} else if (fieldName.equals("format")) {
340356
format = fieldNode.toString();
357+
} else if (fieldName.equals("default")) {
358+
defaultTimestamp = fieldNode.toString();
341359
}
342360
}
343-
this.timestamp = new Timestamp(enabled, path, format);
361+
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
344362
} else {
345363
this.timestamp = Timestamp.EMPTY;
346364
}
@@ -528,6 +546,14 @@ public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws I
528546
out.writeBoolean(false);
529547
}
530548
out.writeString(mappingMd.timestamp().format());
549+
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
550+
if (mappingMd.timestamp().hasDefaultTimestamp()) {
551+
out.writeBoolean(true);
552+
out.writeString(mappingMd.timestamp().defaultTimestamp());
553+
} else {
554+
out.writeBoolean(false);
555+
}
556+
}
531557
out.writeBoolean(mappingMd.hasParentField());
532558
}
533559

@@ -565,7 +591,8 @@ public static MappingMetaData readFrom(StreamInput in) throws IOException {
565591
// routing
566592
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
567593
// timestamp
568-
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString());
594+
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(),
595+
in.getVersion().onOrAfter(Version.V_1_4_0) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
569596
final boolean hasParentField = in.readBoolean();
570597
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
571598
}

‎src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,15 @@ public static class Defaults extends DateFieldMapper.Defaults {
7070
public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
7171
public static final String PATH = null;
7272
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
73+
public static final String DEFAULT_TIMESTAMP = "now";
7374
}
7475

7576
public static class Builder extends NumberFieldMapper.Builder<Builder, TimestampFieldMapper> {
7677

7778
private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
7879
private String path = Defaults.PATH;
7980
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
81+
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
8082

8183
public Builder() {
8284
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@@ -97,14 +99,19 @@ public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
9799
return builder;
98100
}
99101

102+
public Builder defaultTimestamp(String defaultTimestamp) {
103+
this.defaultTimestamp = defaultTimestamp;
104+
return builder;
105+
}
106+
100107
@Override
101108
public TimestampFieldMapper build(BuilderContext context) {
102109
boolean roundCeil = Defaults.ROUND_CEIL;
103110
if (context.indexSettings() != null) {
104111
Settings settings = context.indexSettings();
105112
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
106113
}
107-
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil,
114+
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
108115
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
109116
}
110117
}
@@ -124,6 +131,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
124131
builder.path(fieldNode.toString());
125132
} else if (fieldName.equals("format")) {
126133
builder.dateTimeFormatter(parseDateTimeFormatter(builder.name(), fieldNode.toString()));
134+
} else if (fieldName.equals("default")) {
135+
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
127136
}
128137
}
129138
return builder;
@@ -134,15 +143,16 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
134143
private EnabledAttributeMapper enabledState;
135144

136145
private final String path;
146+
private final String defaultTimestamp;
137147

138148
public TimestampFieldMapper() {
139-
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER,
149+
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
140150
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY);
141151
}
142152

143153
protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
144-
FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil,
145-
Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
154+
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
155+
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
146156
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
147157
@Nullable Settings fieldDataSettings, Settings indexSettings) {
148158
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
@@ -152,6 +162,7 @@ protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAt
152162
indexSettings, MultiFields.empty(), null);
153163
this.enabledState = enabledState;
154164
this.path = path;
165+
this.defaultTimestamp = defaultTimestamp;
155166
}
156167

157168
@Override
@@ -167,6 +178,10 @@ public String path() {
167178
return this.path;
168179
}
169180

181+
public String defaultTimestamp() {
182+
return this.defaultTimestamp;
183+
}
184+
170185
public FormatDateTimeFormatter dateTimeFormatter() {
171186
return this.dateTimeFormatter;
172187
}
@@ -226,7 +241,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
226241
// if all are defaults, no sense to write it at all
227242
if (!includeDefaults && fieldType.indexed() == Defaults.FIELD_TYPE.indexed() && customFieldDataSettings == null &&
228243
fieldType.stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED && path == Defaults.PATH
229-
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
244+
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())
245+
&& Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
230246
return builder;
231247
}
232248
builder.startObject(CONTENT_TYPE);
@@ -246,6 +262,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
246262
if (includeDefaults || !dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
247263
builder.field("format", dateTimeFormatter.format());
248264
}
265+
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
266+
builder.field("default", defaultTimestamp);
267+
}
249268
if (customFieldDataSettings != null) {
250269
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
251270
} else if (includeDefaults) {

0 commit comments

Comments
 (0)