Skip to content

Commit fd76291

Browse files
johtanijohtani
committed
Index Template: parse and validate mappings in template when template puts
Share applying template with MetaDataCreateIndexService and MetaDataIndexTemplateService Add some unit test Collapse addMappingsToMapperService and move it to MapperService Extract validateTemplate method use expectThrows in testcase Add TODO comment Closes #2415
1 parent 93c5a9d commit fd76291

File tree

8 files changed

+359
-113
lines changed

8 files changed

+359
-113
lines changed

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java‎

Lines changed: 8 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,7 @@
5555
import org.elasticsearch.common.regex.Regex;
5656
import org.elasticsearch.common.settings.IndexScopedSettings;
5757
import org.elasticsearch.common.settings.Settings;
58-
import org.elasticsearch.common.xcontent.XContentFactory;
5958
import org.elasticsearch.common.xcontent.XContentHelper;
60-
import org.elasticsearch.common.xcontent.XContentParser;
6159
import org.elasticsearch.env.Environment;
6260
import org.elasticsearch.index.Index;
6361
import org.elasticsearch.index.IndexNotFoundException;
@@ -220,7 +218,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
220218
List<String> templateNames = new ArrayList<>();
221219

222220
for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
223-
mappings.put(entry.getKey(), parseMapping(entry.getValue()));
221+
mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
224222
}
225223

226224
for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
@@ -232,9 +230,9 @@ public ClusterState execute(ClusterState currentState) throws Exception {
232230
templateNames.add(template.getName());
233231
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
234232
if (mappings.containsKey(cursor.key)) {
235-
XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string()));
233+
XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
236234
} else {
237-
mappings.put(cursor.key, parseMapping(cursor.value.string()));
235+
mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
238236
}
239237
}
240238
// handle custom
@@ -315,26 +313,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
315313
createdIndex = indexService.index();
316314
// now add the mappings
317315
MapperService mapperService = indexService.mapperService();
318-
// first, add the default mapping
319-
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
320-
try {
321-
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
322-
} catch (Exception e) {
323-
removalReason = "failed on parsing default mapping on index creation";
324-
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage());
325-
}
326-
}
327-
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
328-
if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
329-
continue;
330-
}
331-
try {
332-
// apply the default here, its the first time we parse it
333-
mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
334-
} catch (Exception e) {
335-
removalReason = "failed on parsing mappings on index creation";
336-
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
337-
}
316+
try {
317+
mapperService.merge(mappings, request.updateAllTypes());
318+
} catch (MapperParsingException mpe) {
319+
removalReason = "failed on parsing default mapping/mappings on index creation";
320+
throw mpe;
338321
}
339322

340323
final QueryShardContext queryShardContext = indexService.newQueryShardContext();
@@ -426,12 +409,6 @@ public ClusterState execute(ClusterState currentState) throws Exception {
426409
});
427410
}
428411

429-
private Map<String, Object> parseMapping(String mappingSource) throws Exception {
430-
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
431-
return parser.map();
432-
}
433-
}
434-
435412
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
436413
List<IndexTemplateMetaData> templates = new ArrayList<>();
437414
for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java‎

Lines changed: 72 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,33 @@
1919
package org.elasticsearch.cluster.metadata;
2020

2121
import com.carrotsearch.hppc.cursors.ObjectCursor;
22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.action.admin.indices.alias.Alias;
2324
import org.elasticsearch.action.support.master.MasterNodeRequest;
2425
import org.elasticsearch.cluster.ClusterState;
2526
import org.elasticsearch.cluster.ClusterStateUpdateTask;
2627
import org.elasticsearch.cluster.service.ClusterService;
2728
import org.elasticsearch.common.Priority;
2829
import org.elasticsearch.common.Strings;
30+
import org.elasticsearch.common.UUIDs;
2931
import org.elasticsearch.common.ValidationException;
3032
import org.elasticsearch.common.component.AbstractComponent;
3133
import org.elasticsearch.common.inject.Inject;
3234
import org.elasticsearch.common.regex.Regex;
3335
import org.elasticsearch.common.settings.Settings;
3436
import org.elasticsearch.common.unit.TimeValue;
37+
import org.elasticsearch.index.Index;
38+
import org.elasticsearch.index.NodeServicesProvider;
39+
import org.elasticsearch.index.IndexService;
40+
import org.elasticsearch.index.mapper.MapperParsingException;
41+
import org.elasticsearch.index.mapper.MapperService;
3542
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
3643
import org.elasticsearch.indices.IndexTemplateMissingException;
44+
import org.elasticsearch.indices.IndicesService;
3745
import org.elasticsearch.indices.InvalidIndexTemplateException;
3846

3947
import java.util.ArrayList;
48+
import java.util.Collections;
4049
import java.util.HashMap;
4150
import java.util.HashSet;
4251
import java.util.List;
@@ -51,14 +60,18 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
5160

5261
private final ClusterService clusterService;
5362
private final AliasValidator aliasValidator;
63+
private final IndicesService indicesService;
5464
private final MetaDataCreateIndexService metaDataCreateIndexService;
65+
private final NodeServicesProvider nodeServicesProvider;
5566

5667
@Inject
57-
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator) {
68+
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
5869
super(settings);
5970
this.clusterService = clusterService;
6071
this.aliasValidator = aliasValidator;
72+
this.indicesService = indicesService;
6173
this.metaDataCreateIndexService = metaDataCreateIndexService;
74+
this.nodeServicesProvider = nodeServicesProvider;
6275
}
6376

6477
public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
@@ -126,28 +139,7 @@ public void putTemplate(final PutRequest request, final PutListener listener) {
126139
return;
127140
}
128141

129-
IndexTemplateMetaData.Builder templateBuilder;
130-
try {
131-
templateBuilder = IndexTemplateMetaData.builder(request.name);
132-
templateBuilder.order(request.order);
133-
templateBuilder.template(request.template);
134-
templateBuilder.settings(request.settings);
135-
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
136-
templateBuilder.putMapping(entry.getKey(), entry.getValue());
137-
}
138-
for (Alias alias : request.aliases) {
139-
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
140-
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
141-
templateBuilder.putAlias(aliasMetaData);
142-
}
143-
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
144-
templateBuilder.putCustom(entry.getKey(), entry.getValue());
145-
}
146-
} catch (Throwable e) {
147-
listener.onFailure(e);
148-
return;
149-
}
150-
final IndexTemplateMetaData template = templateBuilder.build();
142+
final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);
151143

152144
clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]",
153145
new ClusterStateUpdateTask(Priority.URGENT) {
@@ -163,22 +155,77 @@ public void onFailure(String source, Throwable t) {
163155
}
164156

165157
@Override
166-
public ClusterState execute(ClusterState currentState) {
158+
public ClusterState execute(ClusterState currentState) throws Exception {
167159
if (request.create && currentState.metaData().templates().containsKey(request.name)) {
168160
throw new IndexTemplateAlreadyExistsException(request.name);
169161
}
162+
163+
validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider, metaDataCreateIndexService);
164+
165+
for (Alias alias : request.aliases) {
166+
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
167+
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
168+
templateBuilder.putAlias(aliasMetaData);
169+
}
170+
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
171+
templateBuilder.putCustom(entry.getKey(), entry.getValue());
172+
}
173+
IndexTemplateMetaData template = templateBuilder.build();
174+
170175
MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);
171176

172177
return ClusterState.builder(currentState).metaData(builder).build();
173178
}
174179

175180
@Override
176181
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
177-
listener.onResponse(new PutResponse(true, template));
182+
listener.onResponse(new PutResponse(true, templateBuilder.build()));
178183
}
179184
});
180185
}
181186

187+
private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
188+
NodeServicesProvider nodeServicesProvider, MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
189+
Index createdIndex = null;
190+
final String temporaryIndexName = UUIDs.randomBase64UUID();
191+
try {
192+
193+
//create index service for parsing and validating "mappings"
194+
Settings dummySettings = Settings.builder()
195+
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
196+
.put(request.settings)
197+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
198+
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
199+
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
200+
.build();
201+
202+
final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
203+
IndexService dummyIndexService = indicesService.createIndex(nodeServicesProvider, tmpIndexMetadata, Collections.emptyList());
204+
createdIndex = dummyIndexService.index();
205+
206+
templateBuilder.order(request.order);
207+
templateBuilder.template(request.template);
208+
templateBuilder.settings(request.settings);
209+
210+
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
211+
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
212+
try {
213+
templateBuilder.putMapping(entry.getKey(), entry.getValue());
214+
} catch (Exception e) {
215+
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
216+
}
217+
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
218+
}
219+
220+
dummyIndexService.mapperService().merge(mappingsForValidation, false);
221+
222+
} finally {
223+
if (createdIndex != null) {
224+
indicesService.removeIndex(createdIndex, " created for parsing template mapping");
225+
}
226+
}
227+
}
228+
182229
private void validate(PutRequest request) {
183230
List<String> validationErrors = new ArrayList<>();
184231
if (request.name.contains(" ")) {

‎core/src/main/java/org/elasticsearch/index/mapper/MapperService.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.elasticsearch.common.regex.Regex;
3030
import org.elasticsearch.common.settings.Setting;
3131
import org.elasticsearch.common.settings.Setting.Property;
32+
import org.elasticsearch.common.xcontent.XContentFactory;
33+
import org.elasticsearch.common.xcontent.XContentParser;
3234
import org.elasticsearch.index.AbstractIndexComponent;
3335
import org.elasticsearch.index.IndexSettings;
3436
import org.elasticsearch.index.analysis.AnalysisService;
@@ -175,6 +177,35 @@ public DocumentMapperParser documentMapperParser() {
175177
return this.documentParser;
176178
}
177179

180+
public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
181+
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
182+
return parser.map();
183+
}
184+
}
185+
186+
//TODO: make this atomic
187+
public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
188+
// first, add the default mapping
189+
if (mappings.containsKey(DEFAULT_MAPPING)) {
190+
try {
191+
this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
192+
} catch (Exception e) {
193+
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
194+
}
195+
}
196+
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
197+
if (entry.getKey().equals(DEFAULT_MAPPING)) {
198+
continue;
199+
}
200+
try {
201+
// apply the default here, its the first time we parse it
202+
this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
203+
} catch (Exception e) {
204+
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
205+
}
206+
}
207+
}
208+
178209
public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
179210
if (DEFAULT_MAPPING.equals(type)) {
180211
// verify we can parse it

0 commit comments

Comments
 (0)