Skip to content

Add new default option for timestamp field #7036

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Add new default option for timestamp field #7036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
472fc35
Generate timestamp when path is null
dadoonet Jul 25, 2014
10b4473
Update after @jpountz review
dadoonet Jul 28, 2014
bb0d5f2
Update after @jpountz 2nd review
dadoonet Jul 29, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 45 additions & 1 deletion docs/reference/mapping/fields/timestamp-field.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
The `_timestamp` field allows to automatically index the timestamp of a
document. It can be provided externally via the index request or in the
`_source`. If it is not provided externally it will be automatically set
to the date the document was processed by the indexing chain.
to a <<mapping-timestamp-field-default,default date>>.

[float]
==== enabled
Expand Down Expand Up @@ -60,6 +60,7 @@ Note, using `path` without explicit timestamp value provided requires an
additional (though quite fast) parsing phase.

[float]
[[mapping-timestamp-field-format]]
==== format

You can define the <<mapping-date-format,date
Expand All @@ -80,3 +81,46 @@ format>> used to parse the provided timestamp value. For example:

Note, the default format is `dateOptionalTime`. The timestamp value will
first be parsed as a number and if it fails the format will be tried.

[float]
[[mapping-timestamp-field-default]]
==== default

You can define a default value for when timestamp is not provided
within the index request or in the `_source` document.

By default, the default value is `now` which means the date the document was processed by the indexing chain.

You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:

[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"default" : null
}
}
}
--------------------------------------------------

If you don't provide any timestamp value, indexation will fail.

You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:

[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"format" : "YYYY-MM-dd",
"default" : "1970-01-01"
}
}
}
--------------------------------------------------

If you don't provide any timestamp value, indexation will fail.

18 changes: 16 additions & 2 deletions src/main/java/org/elasticsearch/action/index/IndexRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.elasticsearch.*;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.RoutingMissingException;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.MappingMetaData;
Expand Down Expand Up @@ -574,7 +575,9 @@ public void process(MetaData metaData, String aliasOrIndex, @Nullable MappingMet
}
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
if (timestamp != null) {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
}
}
} catch (MapperParsingException e) {
throw e;
Expand Down Expand Up @@ -613,7 +616,18 @@ public void process(MetaData metaData, String aliasOrIndex, @Nullable MappingMet

// generate timestamp if not provided, we always have one post this stage...
if (timestamp == null) {
timestamp = Long.toString(System.currentTimeMillis());
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
if (mappingMd != null && mappingMd.timestamp() != null) {
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
}
if (!Strings.hasText(defaultTimestamp)) {
throw new TimestampParsingException("timestamp is required by mapping");
}
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
}
}
}

Expand Down
37 changes: 32 additions & 5 deletions src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.cluster.metadata;

import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
Expand Down Expand Up @@ -175,7 +176,8 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
}


public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT);
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);

private final boolean enabled;

Expand All @@ -187,7 +189,9 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi

private final FormatDateTimeFormatter dateTimeFormatter;

public Timestamp(boolean enabled, String path, String format) {
private final String defaultTimestamp;

public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
this.enabled = enabled;
this.path = path;
if (path == null) {
Expand All @@ -197,6 +201,7 @@ public Timestamp(boolean enabled, String path, String format) {
}
this.format = format;
this.dateTimeFormatter = Joda.forPattern(format);
this.defaultTimestamp = defaultTimestamp;
}

public boolean enabled() {
Expand All @@ -219,6 +224,14 @@ public String format() {
return this.format;
}

public String defaultTimestamp() {
return this.defaultTimestamp;
}

public boolean hasDefaultTimestamp() {
return this.defaultTimestamp != null;
}

public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
Expand All @@ -233,6 +246,7 @@ public boolean equals(Object o) {
if (enabled != timestamp.enabled) return false;
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;

return true;
Expand All @@ -245,6 +259,7 @@ public int hashCode() {
result = 31 * result + (format != null ? format.hashCode() : 0);
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
return result;
}
}
Expand All @@ -263,7 +278,7 @@ public MappingMetaData(DocumentMapper docMapper) {
this.source = docMapper.mappingSource();
this.id = new Id(docMapper.idFieldMapper().path());
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
this.hasParentField = docMapper.parentFieldMapper().active();
}

Expand Down Expand Up @@ -328,6 +343,7 @@ private void initMappers(Map<String, Object> withoutType) {
boolean enabled = false;
String path = null;
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Expand All @@ -338,9 +354,11 @@ private void initMappers(Map<String, Object> withoutType) {
path = fieldNode.toString();
} else if (fieldName.equals("format")) {
format = fieldNode.toString();
} else if (fieldName.equals("default")) {
defaultTimestamp = fieldNode.toString();
}
}
this.timestamp = new Timestamp(enabled, path, format);
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
} else {
this.timestamp = Timestamp.EMPTY;
}
Expand Down Expand Up @@ -528,6 +546,14 @@ public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws I
out.writeBoolean(false);
}
out.writeString(mappingMd.timestamp().format());
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
if (mappingMd.timestamp().hasDefaultTimestamp()) {
out.writeBoolean(true);
out.writeString(mappingMd.timestamp().defaultTimestamp());
} else {
out.writeBoolean(false);
}
}
out.writeBoolean(mappingMd.hasParentField());
}

Expand Down Expand Up @@ -565,7 +591,8 @@ public static MappingMetaData readFrom(StreamInput in) throws IOException {
// routing
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
// timestamp
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString());
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(),
in.getVersion().onOrAfter(Version.V_1_4_0) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
final boolean hasParentField = in.readBoolean();
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
}
Expand Down
29 changes: 24 additions & 5 deletions src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,15 @@ public static class Defaults extends DateFieldMapper.Defaults {
public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
public static final String PATH = null;
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
public static final String DEFAULT_TIMESTAMP = "now";
}

public static class Builder extends NumberFieldMapper.Builder<Builder, TimestampFieldMapper> {

private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
private String path = Defaults.PATH;
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;

public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
Expand All @@ -97,14 +99,19 @@ public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
return builder;
}

public Builder defaultTimestamp(String defaultTimestamp) {
this.defaultTimestamp = defaultTimestamp;
return builder;
}

@Override
public TimestampFieldMapper build(BuilderContext context) {
boolean roundCeil = Defaults.ROUND_CEIL;
if (context.indexSettings() != null) {
Settings settings = context.indexSettings();
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil,
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
Expand All @@ -124,6 +131,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
builder.path(fieldNode.toString());
} else if (fieldName.equals("format")) {
builder.dateTimeFormatter(parseDateTimeFormatter(builder.name(), fieldNode.toString()));
} else if (fieldName.equals("default")) {
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
}
}
return builder;
Expand All @@ -134,15 +143,16 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
private EnabledAttributeMapper enabledState;

private final String path;
private final String defaultTimestamp;

public TimestampFieldMapper() {
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER,
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY);
}

protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil,
Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Expand All @@ -152,6 +162,7 @@ protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAt
indexSettings, MultiFields.empty(), null);
this.enabledState = enabledState;
this.path = path;
this.defaultTimestamp = defaultTimestamp;
}

@Override
Expand All @@ -167,6 +178,10 @@ public String path() {
return this.path;
}

public String defaultTimestamp() {
return this.defaultTimestamp;
}

public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
Expand Down Expand Up @@ -226,7 +241,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
// if all are defaults, no sense to write it at all
if (!includeDefaults && fieldType.indexed() == Defaults.FIELD_TYPE.indexed() && customFieldDataSettings == null &&
fieldType.stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED && path == Defaults.PATH
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())
&& Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
return builder;
}
builder.startObject(CONTENT_TYPE);
Expand All @@ -246,6 +262,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (includeDefaults || !dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
builder.field("format", dateTimeFormatter.format());
}
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
builder.field("default", defaultTimestamp);
}
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {
Expand Down
Loading