diff --git a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java index c584c8856a5b5..63fe321985c1c 100644 --- a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java +++ b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java @@ -143,7 +143,7 @@ protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) { throw new IllegalArgumentException("suggest content missing"); } final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(), - indexService.queryParserService(), request.shardId().getIndex(), request.shardId().id()); + indexService.queryParserService(), indexService.fieldData(), request.shardId().getIndex(), request.shardId().id()); final Suggest result = suggestPhase.execute(context, searcher.searcher()); return new ShardSuggestResponse(request.shardId(), result); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java index 98e450d1265cf..3baa413c285b6 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java @@ -21,10 +21,11 @@ import java.io.IOException; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; public interface SuggestContextParser { - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService) throws IOException; + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, IndexFieldDataService indexFieldDataService) throws IOException; } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java index 74ddf6a049812..013a54d0e1550 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java @@ -21,6 +21,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.search.SearchParseElement; @@ -47,11 +48,11 @@ public SuggestParseElement(Suggesters suggesters) { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.queryParserService(), context.shardTarget().index(), context.shardTarget().shardId()); + SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.queryParserService(), context.fieldData(), context.shardTarget().index(), context.shardTarget().shardId()); context.suggest(suggestionSearchContext); } - public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, String index, int shardId) throws IOException { + public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, IndexFieldDataService fieldDataService, String index, int shardId) throws IOException { SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); BytesRef globalText = null; @@ -99,7 +100,7 @@ public SuggestionSearchContext parseInternal(XContentParser parser, MapperServic if (contextParser instanceof CompletionSuggestParser) { ((CompletionSuggestParser) contextParser).setOldCompletionSuggester(((CompletionSuggester) suggesters.get("completion_old"))); } - suggestionContext = contextParser.parse(parser, mapperService, queryParserService); + suggestionContext = contextParser.parse(parser, mapperService, queryParserService, fieldDataService); } } if (suggestionContext != null) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java index c861e104279be..e966e6c899246 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; @@ -38,9 +39,7 @@ import org.elasticsearch.search.suggest.completion.context.ContextMappingsParser; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.elasticsearch.search.suggest.SuggestUtils.parseSuggestContext; import static org.elasticsearch.search.suggest.completion.context.ContextMappingsParser.parseQueryContext; @@ -87,7 +86,7 @@ public CompletionSuggestParser(CompletionSuggester completionSuggester) { @Override public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, - IndexQueryParserService queryParserService) throws IOException { + IndexQueryParserService queryParserService, IndexFieldDataService fieldDataService) throws IOException { XContentParser.Token token; String fieldName = null; CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester); @@ -95,6 +94,7 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma XContentParser contextParser = null; CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptions = null; CompletionSuggestionBuilder.RegexOptionsBuilder regexOptions = null; + Set payloadFields = new HashSet<>(1); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -105,6 +105,8 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma if (parser.booleanValue()) { fuzzyOptions = new CompletionSuggestionBuilder.FuzzyOptionsBuilder(); } + } else if (token == XContentParser.Token.VALUE_STRING && "payload".equals(fieldName)) { + payloadFields.add(parser.text()); } } } else if (token == XContentParser.Token.START_OBJECT) { @@ -158,6 +160,18 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma } else { throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]"); } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("payload".equals(fieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + payloadFields.add(parser.text()); + } else { + throw new IllegalArgumentException("suggester [completion] expected string values in [payload] array"); + } + } + } else { + throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]"); + } } else { throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]"); } @@ -177,10 +191,13 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma contextParser.close(); } - suggestion.fieldType(type); + suggestion.setFieldType(type); suggestion.setFuzzyOptionsBuilder(fuzzyOptions); suggestion.setRegexOptionsBuilder(regexOptions); suggestion.setQueryContexts(queryContexts); + suggestion.setMapperService(mapperService); + suggestion.setFieldData(fieldDataService); + suggestion.setPayloadFields(payloadFields); // TODO: pass a query builder or the query itself? // now we do it in CompletionSuggester#toQuery(CompletionSuggestionContext) return suggestion; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index 3cbfcee6b3aae..22f317733b97a 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.suggest.completion; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.IndexSearcher; @@ -30,6 +31,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.fielddata.AtomicFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestContextParser; @@ -50,7 +54,7 @@ public SuggestContextParser getContextParser() { @Override protected Suggest.Suggestion> innerExecute(String name, CompletionSuggestionContext suggestionContext, IndexSearcher searcher, CharsRefBuilder spare) throws IOException { - if (suggestionContext.fieldType() == null) { + if (suggestionContext.getFieldType() == null) { throw new ElasticsearchException("Field [" + suggestionContext.getField() + "] is not a completion suggest field"); } CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize()); @@ -67,18 +71,38 @@ protected Suggest.Suggestion contextEntry; - if (suggestionContext.fieldType().hasContextMappings() && suggestDoc.context != null) { - contextEntry = suggestionContext.fieldType().getContextMappings().getNamedContext(suggestDoc.context); + if (suggestionContext.getFieldType().hasContextMappings() && suggestDoc.context != null) { + contextEntry = suggestionContext.getFieldType().getContextMappings().getNamedContext(suggestDoc.context); } else { assert suggestDoc.context == null; contextEntry = null; } - final Option value = results.get(suggestDoc.doc); + final CompletionSuggestion.Entry.Option value = results.get(suggestDoc.doc); if (value == null) { - final Option option = new Option(suggestDoc.doc, new StringText(key), score, contextEntry); + final Map> payload; + Set payloadFields = suggestionContext.getPayloadFields(); + if (!payloadFields.isEmpty()) { + int readerIndex = ReaderUtil.subIndex(suggestDoc.doc, searcher.getIndexReader().leaves()); + LeafReaderContext subReaderContext = searcher.getIndexReader().leaves().get(readerIndex); + int subDocId = suggestDoc.doc - subReaderContext.docBase; + payload = new LinkedHashMap<>(payloadFields.size()); + for (String field : payloadFields) { + MappedFieldType fieldType = suggestionContext.getMapperService().smartNameFieldType(field); + if (fieldType != null) { + AtomicFieldData data = suggestionContext.getFieldData().getForField(fieldType).load(subReaderContext); + ScriptDocValues scriptValues = data.getScriptValues(); + scriptValues.setNextDocId(subDocId); + payload.put(field, scriptValues.getValues()); + } else { + throw new ElasticsearchException("Payload field [" + field + "] does not exist"); + } + } + } else { + payload = Collections.emptyMap(); + } + final CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(new StringText(suggestDoc.key.toString()), suggestDoc.score, contextEntry, payload); results.put(suggestDoc.doc, option); } else { value.addContextEntry(contextEntry); @@ -100,7 +124,7 @@ protected Suggest.Suggestion { + + public OptionPriorityQueue(int maxSize) { + super(maxSize); + } + + @Override + protected boolean lessThan(Entry.Option a, Entry.Option b) { + return sortComparator().compare(a, b) > 0; + } + + public Entry.Option[] get() { + int size = size(); + Entry.Option[] results = new Entry.Option[size]; + for (int i = size - 1; i >= 0; i--) { + results[i] = pop(); + } + return results; + } + } + + @Override + public Suggest.Suggestion reduce(List> toReduce) { + if (toReduce.size() == 1) { + return toReduce.get(0); + } else { + // combine suggestion entries from participating shards on the coordinating node + // the global top size entries are collected from the shard results + // using a priority queue + OptionPriorityQueue priorityQueue = new OptionPriorityQueue(size); + for (Suggest.Suggestion entries : toReduce) { + assert entries.getEntries().size() == 1 : "CompletionSuggestion must have only one entry"; + for (Entry.Option option : entries.getEntries().get(0)) { + if (option == priorityQueue.insertWithOverflow(option)) { + // if the current option has overflown from pq, + // we can assume all of the successive options + // from this shard result will be overflown as well + break; + } + } + } + Entry options = this.entries.get(0); + options.getOptions().clear(); + Collections.addAll(options.getOptions(), priorityQueue.get()); + return this; + } + } + @Override public int getType() { return TYPE; @@ -81,11 +129,11 @@ protected Option newOption() { public static class Option extends Suggest.Suggestion.Entry.Option { private Map> contexts = new TreeMap<>(); - private int docID; + private Map> payload; - public Option(int docID, Text text, float score, Map.Entry contextEntry) { + public Option(Text text, float score, Map.Entry contextEntry, Map> payload) { super(text, score); - this.docID = docID; + this.payload = payload; addContextEntry(contextEntry); } @@ -95,8 +143,9 @@ protected Option() { @Override protected void mergeInto(Suggest.Suggestion.Entry.Option otherOption) { - super.mergeInto(otherOption); - this.contexts.putAll(((Option) otherOption).contexts); + // Completion suggestions are reduced by + // org.elasticsearch.search.suggest.completion.CompletionSuggestion.reduce() + throw new UnsupportedOperationException(); } public void addContextEntry(Map.Entry entry) { @@ -115,8 +164,8 @@ public void addContextEntry(Map.Entry entry) { } } - int getDocID() { - return docID; + public Map> getPayload() { + return payload; } public Map> getContexts() { @@ -131,6 +180,17 @@ public void setScore(float score) { @Override protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { super.innerToXContent(builder, params); + if (payload.size() > 0) { + builder.startObject("payload"); + for (Map.Entry> entry : payload.entrySet()) { + builder.startArray(entry.getKey()); + for (Object payload : entry.getValue()) { + builder.value(payload); + } + builder.endArray(); + } + builder.endObject(); + } if (contexts.size() > 0) { builder.startObject("contexts"); for (Map.Entry> entry : contexts.entrySet()) { @@ -148,9 +208,19 @@ protected XContentBuilder innerToXContent(XContentBuilder builder, Params params @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - docID = in.readInt(); - int size = in.readInt(); - for (int i = 0; i < size; i++) { + int payloadSize = in.readInt(); + this.payload = new LinkedHashMap<>(payloadSize); + for (int i = 0; i < payloadSize; i++) { + String payloadName = in.readString(); + int nValues = in.readVInt(); + List values = new ArrayList<>(nValues); + for (int j = 0; j < nValues; j++) { + values.add(in.readGenericValue()); + } + this.payload.put(payloadName, values); + } + int contextSize = in.readInt(); + for (int i = 0; i < contextSize; i++) { String contextName = in.readString(); int nContexts = in.readVInt(); Set contexts = new HashSet<>(nContexts); @@ -164,7 +234,15 @@ public void readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeInt(docID); + out.writeInt(payload.size()); + for (Map.Entry> entry : payload.entrySet()) { + out.writeString(entry.getKey()); + List values = entry.getValue(); + out.writeVInt(values.size()); + for (Object value : values) { + out.writeGenericValue(value); + } + } out.writeInt(contexts.size()); for (Map.Entry> entry : contexts.entrySet()) { out.writeString(entry.getKey()); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java index dedac45d0d016..821941ea0bf19 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java @@ -45,6 +45,7 @@ public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilde private FuzzyOptionsBuilder fuzzyOptionsBuilder; private RegexOptionsBuilder regexOptionsBuilder; private List queryContextsList; + private String[] payloadFields; public CompletionSuggestionBuilder(String name) { super(name, "completion"); @@ -256,6 +257,15 @@ public CompletionSuggestionBuilder regex(String regex, RegexOptionsBuilder regex return this; } + /** + * Sets the fields to be returned as suggestion payload. + * Note: Only doc values enabled fields are supported + */ + public CompletionSuggestionBuilder payload(String... fields) { + this.payloadFields = fields; + return this; + } + /** * Sets query contexts for a category context * @param name of the category context to execute on @@ -289,6 +299,13 @@ private void addQueryContext(String name, T[] queryContex @Override protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + if (payloadFields != null) { + builder.startArray("payload"); + for (String field : payloadFields) { + builder.value(field); + } + builder.endArray(); + } if (fuzzyOptionsBuilder != null) { fuzzyOptionsBuilder.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java index 4e29e66da3b35..31faf311f9364 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java @@ -18,12 +18,15 @@ */ package org.elasticsearch.search.suggest.completion; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import java.util.Map; +import java.util.Set; /** * @@ -34,16 +37,19 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder; private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder; private Map queryContexts; + private MapperService mapperService; + private IndexFieldDataService fieldData; + private Set payloadFields; CompletionSuggestionContext(Suggester suggester) { super(suggester); } - CompletionFieldMapper.CompletionFieldType fieldType() { + CompletionFieldMapper.CompletionFieldType getFieldType() { return this.fieldType; } - void fieldType(CompletionFieldMapper.CompletionFieldType fieldType) { + void setFieldType(CompletionFieldMapper.CompletionFieldType fieldType) { this.fieldType = fieldType; } @@ -70,4 +76,28 @@ void setQueryContexts(Map queryContexts) { Map getQueryContexts() { return queryContexts; } + + void setMapperService(MapperService mapperService) { + this.mapperService = mapperService; + } + + MapperService getMapperService() { + return mapperService; + } + + void setFieldData(IndexFieldDataService fieldData) { + this.fieldData = fieldData; + } + + IndexFieldDataService getFieldData() { + return fieldData; + } + + void setPayloadFields(Set fields) { + this.payloadFields = fields; + } + + Set getPayloadFields() { + return payloadFields; + } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java index 0f6a8096973c2..cf81ae93affcf 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; @@ -48,7 +49,7 @@ public PhraseSuggestParser(PhraseSuggester suggester) { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, IndexFieldDataService indexFieldDataService) throws IOException { PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester); suggestion.setQueryParserService(queryParserService); XContentParser.Token token; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java index 52598e6f3c2c2..f8c1266cc9adb 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; @@ -38,7 +39,7 @@ public TermSuggestParser(TermSuggester suggester) { } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, IndexFieldDataService indexFieldDataService) throws IOException { XContentParser.Token token; String fieldName = null; TermSuggestionContext suggestion = new TermSuggestionContext(suggester); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index efdfe8272b968..d477c92f5819c 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -23,7 +23,10 @@ import org.apache.lucene.analysis.TokenStreamToAutomaton; import org.apache.lucene.search.suggest.xdocument.ContextSuggestField; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; @@ -32,6 +35,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -188,6 +192,132 @@ public void testMixedCompletion() throws Exception { } } + @Test + public void testSuggestWithNumericPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source= jsonBuilder() + .startObject() + .field(FIELD, "suggestion" + i) + .field("count", i) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(numDocs).payload("count"); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); + assertThat(options.getOptions().size(), equalTo(numDocs)); + for (CompletionSuggestion.Entry.Option option : options) { + Map> payloads = option.getPayload(); + assertThat(payloads.keySet(), contains("count")); + } + } + + @Test + public void testMalformedRequestPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + SuggestRequest request = new SuggestRequest(INDEX); + XContentBuilder suggest = jsonBuilder().startObject() + .startObject("bad-payload") + .field("prefix", "sug") + .startObject("completion") + .field("field", FIELD) + .startArray("payload") + .startObject() + .field("payload", "field") + .endObject() + .endArray() + .endObject() + .endObject().endObject(); + request.suggest(suggest.bytes()); + ensureYellow(); + + SuggestResponse suggestResponse = client().suggest(request).get(); + assertThat(suggestResponse.getSuccessfulShards(), equalTo(0)); + for (ShardOperationFailedException exception : suggestResponse.getShardFailures()) { + Throwable unwrap = ExceptionsHelper.unwrap(exception.getCause(), IllegalArgumentException.class); + assertNotNull(unwrap); + assertThat(unwrap.getMessage(), containsString("payload")); + } + } + + @Test + public void testMissingPayloadField() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + List indexRequestBuilders = Arrays.asList( + client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "suggestion", "test_field", "test"), + client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, "suggestion") + ); + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").payload("test_field"); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); + assertThat(options.getOptions().size(), equalTo(2)); + for (CompletionSuggestion.Entry.Option option : options.getOptions()) { + assertThat(option.getPayload().keySet(), contains("test_field")); + } + } + + @Test + public void testSuggestWithPayload() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = randomIntBetween(10, 100); + int numPayloadFields = randomIntBetween(2, 5); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i) + .endObject(); + for (int j = 0; j < numPayloadFields; j++) { + source.field("test_field" + j, j + "value" + i); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + int suggestionSize = randomIntBetween(1, numDocs); + int numRequestedPayloadFields = randomIntBetween(2, numPayloadFields); + String[] payloadFields = new String[numRequestedPayloadFields]; + for (int i = 0; i < numRequestedPayloadFields; i++) { + payloadFields[i] = "test_field" + i; + } + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(suggestionSize).payload(payloadFields); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(prefix).execute().actionGet(); + assertNoFailures(suggestResponse); + CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); + assertThat(options.getOptions().size(), equalTo(suggestionSize)); + int id = numDocs; + for (CompletionSuggestion.Entry.Option option : options) { + assertThat(option.getText().toString(), equalTo("suggestion" + id)); + assertThat(option.getPayload().size(), equalTo(numRequestedPayloadFields)); + for (int i = 0; i < numRequestedPayloadFields; i++) { + List fieldValue = option.getPayload().get("test_field" + i); + assertNotNull(fieldValue); + assertThat(fieldValue.size(), equalTo(1)); + assertThat((String)fieldValue.get(0), equalTo(i + "value" + id)); + } + id--; + } + } + @Test public void testSuggestFieldWithPercolateApi() throws Exception { createIndexAndMapping(completionMappingBuilder); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java index e3dfe3b96d3e4..b311c9939c7b1 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java @@ -22,6 +22,7 @@ import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; @@ -59,7 +60,7 @@ public Suggest.Suggestion options = parser.map(); CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options); suggestionContext.setField((String) options.get("field")); diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index 17f82c1ea4f62..2f027c5c9c2e7 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -109,7 +109,9 @@ a weight with suggestion(s). ==== Querying Suggesting works as usual, except that you have to specify the suggest -type as `completion`. +type as `completion`. Suggestions are near real-time, which means +new suggestions can be made visible by <> and +documents once deleted are never shown. [source,js] -------------------------------------------------- @@ -144,17 +146,74 @@ POST music/_suggest?pretty The configured weight for a suggestion is returned as `score`. The `text` field uses the `input` of your indexed suggestion. -The basic completion suggester query supports the following two parameters: +Suggestions are document oriented, you can specify fields to be +returned as part of suggestion payload. All field types (`string`, +`numeric`, `date`, etc) are supported. + +For example, if you index a "title" field along with the suggestion +as follows: + +[source,js] +-------------------------------------------------- +POST music/song +{ + "suggest" : "Nirvana", + "title" : "Nevermind" +} +-------------------------------------------------- + +You can get the "title" as part of the suggestion +payload by specifying it as a `payload`: + +[source,js] +-------------------------------------------------- +POST music/_suggest?pretty +{ + "song-suggest" : { + "prefix" : "n", + "completion" : { + "field" : "suggest" + "payload" : [ "title" ] <1> + } + } +} + +{ + "_shards" : { + "total" : 5, + "successful" : 5, + "failed" : 0 + }, + "song-suggest" : [ { + "text" : "n", + "offset" : 0, + "length" : 1, + "options" : [ { + "text" : "Nirvana", + "score" : 34.0, + "payload" : { + "title" : [ "Nevermind" ] + } + } ] + } ] +} +-------------------------------------------------- +<1> The fields to be returned as part of each suggestion payload. + +The basic completion suggester query supports the following parameters: `field`:: The name of the field on which to run the query (required). `size`:: The number of suggestions to return (defaults to `5`). +`payload`:: The name of the field or field name array to be returned + as payload (defaults to no fields). NOTE: The completion suggester considers all documents in the index. See <> for an explanation of how to query a subset of documents instead. -NOTE: It will be possible to return the associated documents -with each suggestion in the future (TODO). +NOTE: Specifying `payload` fields will incur additional search performance +hit. The `payload` fields are retrieved eagerly (single pass) for top +suggestions at the shard level using field data or from doc values. [[fuzzy]] ==== Fuzzy queries diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml index 67a9980a2aa56..dc2f38b911c91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml @@ -22,6 +22,11 @@ setup: "type" : "completion" "suggest_5b": "type" : "completion" + "suggest_6": + "type" : "completion" + "count": + "type" : "integer" + "doc_values" : "true" --- "Simple suggestion should work": @@ -227,3 +232,52 @@ setup: - length: { result: 1 } - length: { result.0.options: 1 } - match: { result.0.options.0.text: "baz" } + +--- +"Suggestions with payload fields should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_6: + input: "bar" + weight: 2 + title: "title_bar" + count: 2 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_6: + input: "baz" + weight: 3 + title: "title_baz" + count: 3 + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_6 + payload: [ title, count ] + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "baz" } + - match: { result.0.options.0.payload.title: ["title_baz"] } + - match: { result.0.options.0.payload.count: [3] } + - match: { result.0.options.1.text: "bar" } + - match: { result.0.options.1.payload.title: ["title_bar"] } + - match: { result.0.options.1.payload.count: [2] } +