Skip to content

Commit 15597b4

Browse files
weizijunAdam Locke
weizijun
authored and
Adam Locke
committed
TSDB: Automatically add timestamp mapper (elastic#79136)
If tsdb is enabled we need an `@timestamp` field. This automatically maps the field if it is missing and fails to create indices in time_series mode that map `@timestamp` as anything other than `date` and `date_nanos`.
1 parent 16beb78 commit 15597b4

File tree

17 files changed

+410
-105
lines changed

17 files changed

+410
-105
lines changed

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
2+
---
3+
date:
4+
- skip:
5+
version: " - 7.99.99"
6+
reason: introduced in 8.0.0 to be backported to 7.16.0
7+
8+
- do:
9+
indices.create:
10+
index: test
11+
body:
12+
settings:
13+
index:
14+
mode: time_series
15+
routing_path: [metricset]
16+
number_of_replicas: 0
17+
number_of_shards: 2
18+
mappings:
19+
properties:
20+
"@timestamp":
21+
type: date
22+
metricset:
23+
type: keyword
24+
time_series_dimension: true
25+
26+
- do:
27+
indices.get_mapping:
28+
index: test
29+
- match: { "[email protected]": date }
30+
- match: { 'test.mappings._data_stream_timestamp.enabled': true }
31+
32+
- do:
33+
bulk:
34+
refresh: true
35+
index: test_index
36+
body:
37+
- '{"index": {}}'
38+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}'
39+
40+
- do:
41+
search:
42+
index: test_index
43+
body:
44+
docvalue_fields: [ '@timestamp' ]
45+
- match: {hits.total.value: 1}
46+
- match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] }
47+
48+
---
49+
date_nanos:
50+
- skip:
51+
version: " - 7.99.99"
52+
reason: introduced in 8.0.0 to be backported to 7.16.0
53+
54+
- do:
55+
indices.create:
56+
index: test
57+
body:
58+
settings:
59+
index:
60+
mode: time_series
61+
routing_path: [metricset]
62+
number_of_replicas: 0
63+
number_of_shards: 2
64+
mappings:
65+
properties:
66+
"@timestamp":
67+
type: date_nanos
68+
metricset:
69+
type: keyword
70+
time_series_dimension: true
71+
72+
- do:
73+
indices.get_mapping:
74+
index: test
75+
- match: { "[email protected]": date_nanos }
76+
- match: { 'test.mappings._data_stream_timestamp.enabled': true }
77+
78+
- do:
79+
bulk:
80+
refresh: true
81+
index: test_index
82+
body:
83+
- '{"index": {}}'
84+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}'
85+
86+
- do:
87+
search:
88+
index: test_index
89+
body:
90+
docvalue_fields: [ '@timestamp' ]
91+
- match: {hits.total.value: 1}
92+
- match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] }
93+
94+
---
95+
automatically add with date:
96+
- skip:
97+
version: " - 7.99.99"
98+
reason: introduced in 8.0.0 to be backported to 7.16.0
99+
100+
- do:
101+
indices.create:
102+
index: test
103+
body:
104+
settings:
105+
index:
106+
mode: time_series
107+
routing_path: [metricset]
108+
number_of_replicas: 0
109+
number_of_shards: 2
110+
mappings:
111+
properties:
112+
metricset:
113+
type: keyword
114+
time_series_dimension: true
115+
116+
- do:
117+
indices.get_mapping:
118+
index: test
119+
- match: { 'test.mappings.properties.@timestamp': { "type": date } }
120+
- match: { 'test.mappings._data_stream_timestamp.enabled': true }
121+
122+
- do:
123+
bulk:
124+
refresh: true
125+
index: test_index
126+
body:
127+
- '{"index": {}}'
128+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}'
129+
130+
- do:
131+
search:
132+
index: test_index
133+
body:
134+
docvalue_fields: [ '@timestamp' ]
135+
- match: {hits.total.value: 1}
136+
- match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] }
137+
138+
---
139+
reject @timestamp with wrong type:
140+
- skip:
141+
version: " - 7.99.99"
142+
reason: introduced in 8.0.0 to be backported to 7.16.0
143+
144+
- do:
145+
catch: /data stream timestamp field \[@timestamp\] is of type \[keyword\], but \[date,date_nanos\] is expected/
146+
indices.create:
147+
index: test
148+
body:
149+
settings:
150+
index:
151+
mode: time_series
152+
routing_path: [metricset]
153+
number_of_replicas: 0
154+
number_of_shards: 2
155+
mappings:
156+
properties:
157+
"@timestamp":
158+
type: keyword
159+
160+
---
161+
reject timestamp meta field with wrong type:
162+
- skip:
163+
version: " - 7.99.99"
164+
reason: introduced in 8.0.0 to be backported to 7.16.0
165+
166+
- do:
167+
catch: /.* time series index \[_data_stream_timestamp\] meta field must be enabled/
168+
indices.create:
169+
index: test
170+
body:
171+
settings:
172+
index:
173+
mode: time_series
174+
routing_path: [metricset]
175+
number_of_replicas: 0
176+
number_of_shards: 2
177+
mappings:
178+
_data_stream_timestamp:
179+
enabled: false

‎server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,10 @@
1313
import org.elasticsearch.cluster.metadata.ComponentTemplate;
1414
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1515
import org.elasticsearch.cluster.metadata.Template;
16-
import org.elasticsearch.common.bytes.BytesArray;
1716
import org.elasticsearch.common.compress.CompressedXContent;
18-
import org.elasticsearch.common.xcontent.XContentHelper;
19-
import org.elasticsearch.xcontent.XContentParser;
20-
import org.elasticsearch.xcontent.XContentType;
2117
import org.elasticsearch.test.ESIntegTestCase;
2218

23-
import java.io.IOException;
2419
import java.util.Collections;
25-
import java.util.List;
26-
27-
import static org.hamcrest.Matchers.equalTo;
2820

2921
public class ComposableTemplateIT extends ESIntegTestCase {
3022

@@ -80,23 +72,4 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception {
8072
client().execute(PutComposableIndexTemplateAction.INSTANCE,
8173
new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2)).get();
8274
}
83-
84-
public void testUsageOfDataStreamFails() throws IOException {
85-
// Exception that would happen if a unknown field is provided in a composable template:
86-
// The thrown exception will be used to compare against the exception that is thrown when providing
87-
// a composable template with a data stream definition.
88-
String content = "{\"index_patterns\":[\"logs-*-*\"],\"my_field\":\"bla\"}";
89-
XContentParser parser =
90-
XContentHelper.createParser(xContentRegistry(), null, new BytesArray(content), XContentType.JSON);
91-
Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser));
92-
93-
ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of("logs-*-*"))
94-
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()).build();
95-
Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE,
96-
new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet());
97-
Exception actualException = (Exception) e.getCause();
98-
assertThat(actualException.getMessage(),
99-
equalTo(expectedException.getMessage().replace("[1:32] ", "").replace("my_field", "data_stream")));
100-
assertThat(actualException.getMessage(), equalTo("[index_template] unknown field [data_stream]"));
101-
}
10275
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,9 @@
4242
import org.elasticsearch.xcontent.NamedXContentRegistry;
4343
import org.elasticsearch.xcontent.XContentBuilder;
4444
import org.elasticsearch.xcontent.XContentFactory;
45-
import org.elasticsearch.xcontent.XContentParseException;
4645
import org.elasticsearch.xcontent.XContentType;
4746
import org.elasticsearch.index.Index;
4847
import org.elasticsearch.index.IndexService;
49-
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
5048
import org.elasticsearch.index.mapper.MapperParsingException;
5149
import org.elasticsearch.index.mapper.MapperService;
5250
import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -1231,16 +1229,6 @@ private static void validateCompositeTemplate(final ClusterState state,
12311229
String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName;
12321230
// Parse mappings to ensure they are valid after being composed
12331231

1234-
if (template.getDataStreamTemplate() != null) {
1235-
// If there is no _data_stream meta field mapper and a data stream should be created then
1236-
// fail as if the data_stream field can't be parsed:
1237-
if (tempIndexService.mapperService().isMetadataField(DataStreamTimestampFieldMapper.NAME) == false) {
1238-
// Fail like a parsing expection, since we will be moving data_stream template out of server module and
1239-
// then we would fail with the same error message, like we do here.
1240-
throw new XContentParseException("[index_template] unknown field [data_stream]");
1241-
}
1242-
}
1243-
12441232
List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName, xContentRegistry);
12451233
try {
12461234
MapperService mapperService = tempIndexService.mapperService();

‎server/src/main/java/org/elasticsearch/index/IndexMode.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,21 @@
1111
import org.elasticsearch.cluster.metadata.IndexMetadata;
1212
import org.elasticsearch.common.settings.Setting;
1313
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.common.util.Maps;
1415
import org.elasticsearch.core.Nullable;
16+
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
17+
import org.elasticsearch.index.mapper.DateFieldMapper;
18+
import org.elasticsearch.index.mapper.Mapper;
1519
import org.elasticsearch.index.mapper.MappingLookup;
20+
import org.elasticsearch.index.mapper.MappingParserContext;
21+
import org.elasticsearch.index.mapper.RootObjectMapper;
1622
import org.elasticsearch.index.mapper.RoutingFieldMapper;
1723

24+
import java.util.HashMap;
1825
import java.util.List;
1926
import java.util.Map;
2027
import java.util.Objects;
28+
import java.util.Optional;
2129
import java.util.stream.Stream;
2230

2331
import static java.util.stream.Collectors.toSet;
@@ -44,6 +52,9 @@ void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
4452

4553
@Override
4654
public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {}
55+
56+
@Override
57+
public void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder) {}
4758
},
4859
TIME_SERIES {
4960
@Override
@@ -88,6 +99,47 @@ private String routingRequiredBad() {
8899
private String tsdbMode() {
89100
return "[" + IndexSettings.MODE.getKey() + "=time_series]";
90101
}
102+
103+
@Override
104+
public void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder) {
105+
if (false == mapping.containsKey(DataStreamTimestampFieldMapper.NAME)) {
106+
mapping.put(DataStreamTimestampFieldMapper.NAME, new HashMap<>(Map.of("enabled", true)));
107+
} else {
108+
validateTimeStampField(mapping.get(DataStreamTimestampFieldMapper.NAME));
109+
}
110+
111+
Optional<Mapper.Builder> timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH);
112+
if (timestamp.isEmpty()) {
113+
builder.add(
114+
new DateFieldMapper.Builder(
115+
DataStreamTimestampFieldMapper.DEFAULT_PATH,
116+
DateFieldMapper.Resolution.MILLISECONDS,
117+
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
118+
context.scriptCompiler(),
119+
DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()),
120+
context.getIndexSettings().getIndexVersionCreated(),
121+
context.getIndexSettings().getIndex().getName()
122+
)
123+
);
124+
}
125+
}
126+
127+
private void validateTimeStampField(Object timestampFieldValue) {
128+
if (false == (timestampFieldValue instanceof Map)) {
129+
throw new IllegalArgumentException(
130+
"time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field format error"
131+
);
132+
}
133+
134+
@SuppressWarnings("unchecked")
135+
Map<String, Object> timeStampFieldValueMap = (Map<String, Object>) timestampFieldValue;
136+
if (false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", true))
137+
&& false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", "true"))) {
138+
throw new IllegalArgumentException(
139+
"time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field must be enabled"
140+
);
141+
}
142+
}
91143
};
92144

93145
private static final List<Setting<?>> TIME_SERIES_UNSUPPORTED = List.of(
@@ -115,4 +167,9 @@ private String tsdbMode() {
115167
* Validate aliases targeting this index.
116168
*/
117169
public abstract void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting);
170+
171+
/**
172+
* Validate and/or modify the mappings after after they've been parsed.
173+
*/
174+
public abstract void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder);
118175
}

‎server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@
3232
public class DataStreamTimestampFieldMapper extends MetadataFieldMapper {
3333

3434
public static final String NAME = "_data_stream_timestamp";
35-
private static final String DEFAULT_PATH = "@timestamp";
35+
public static final String DEFAULT_PATH = "@timestamp";
3636

37-
private static final DataStreamTimestampFieldMapper ENABLED_INSTANCE =
37+
public static final DataStreamTimestampFieldMapper ENABLED_INSTANCE =
3838
new DataStreamTimestampFieldMapper(TimestampFieldType.INSTANCE, true);
3939
private static final DataStreamTimestampFieldMapper DISABLED_INSTANCE =
4040
new DataStreamTimestampFieldMapper(TimestampFieldType.INSTANCE, false);

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010

1111
import org.elasticsearch.common.compress.CompressedXContent;
1212
import org.elasticsearch.common.xcontent.XContentHelper;
13-
import org.elasticsearch.xcontent.XContentType;
1413
import org.elasticsearch.core.Nullable;
14+
import org.elasticsearch.xcontent.XContentType;
1515

1616
import java.util.Collections;
1717
import java.util.HashMap;
@@ -94,7 +94,8 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar
9494

9595
private Mapping parse(String type, Map<String, Object> mapping) throws MapperParsingException {
9696
MappingParserContext parserContext = parserContextSupplier.get();
97-
RootObjectMapper rootObjectMapper = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT);
97+
RootObjectMapper.Builder rootObjectMapperBuilder = rootObjectTypeParser.parse(type, mapping, parserContext);
98+
parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping, rootObjectMapperBuilder);
9899

99100
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get();
100101
Map<String, Object> meta = null;
@@ -143,7 +144,7 @@ private Mapping parse(String type, Map<String, Object> mapping) throws MapperPar
143144
checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: ");
144145

145146
return new Mapping(
146-
rootObjectMapper,
147+
rootObjectMapperBuilder.build(MapperBuilderContext.ROOT),
147148
metadataMappers.values().toArray(new MetadataFieldMapper[0]),
148149
meta);
149150
}

0 commit comments

Comments
 (0)