Skip to content

Completion Suggester: Add doc-values based payload support #13659

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
areek merged 1 commit into from
Sep 21, 2015
Merged

Completion Suggester: Add doc-values based payload support #13659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) {
throw new IllegalArgumentException("suggest content missing");
}
final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(),
indexService.queryParserService(), request.shardId().getIndex(), request.shardId().id());
indexService.queryParserService(), indexService.fieldData(), request.shardId().getIndex(), request.shardId().id());
final Suggest result = suggestPhase.execute(context, searcher.searcher());
return new ShardSuggestResponse(request.shardId(), result);
}
Expand Down
3 changes: 2 additions & 1 deletion core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,11 @@
import java.io.IOException;

import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.IndexQueryParserService;

public interface SuggestContextParser {
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService) throws IOException;
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, IndexFieldDataService indexFieldDataService) throws IOException;

}
7 changes: 4 additions & 3 deletions core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.search.SearchParseElement;
Expand All @@ -47,11 +48,11 @@ public SuggestParseElement(Suggesters suggesters) {

@Override
public void parse(XContentParser parser, SearchContext context) throws Exception {
SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.queryParserService(), context.shardTarget().index(), context.shardTarget().shardId());
SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.queryParserService(), context.fieldData(), context.shardTarget().index(), context.shardTarget().shardId());
context.suggest(suggestionSearchContext);
}

public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, String index, int shardId) throws IOException {
public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService, IndexFieldDataService fieldDataService, String index, int shardId) throws IOException {
SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();

BytesRef globalText = null;
Expand Down Expand Up @@ -99,7 +100,7 @@ public SuggestionSearchContext parseInternal(XContentParser parser, MapperServic
if (contextParser instanceof CompletionSuggestParser) {
((CompletionSuggestParser) contextParser).setOldCompletionSuggester(((CompletionSuggester) suggesters.get("completion_old")));
}
suggestionContext = contextParser.parse(parser, mapperService, queryParserService);
suggestionContext = contextParser.parse(parser, mapperService, queryParserService, fieldDataService);
}
}
if (suggestionContext != null) {
Expand Down
27 changes: 22 additions & 5 deletions core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper;
Expand All @@ -38,9 +39,7 @@
import org.elasticsearch.search.suggest.completion.context.ContextMappingsParser;

import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.*;

import static org.elasticsearch.search.suggest.SuggestUtils.parseSuggestContext;
import static org.elasticsearch.search.suggest.completion.context.ContextMappingsParser.parseQueryContext;
Expand Down Expand Up @@ -87,14 +86,15 @@ public CompletionSuggestParser(CompletionSuggester completionSuggester) {

@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService,
IndexQueryParserService queryParserService) throws IOException {
IndexQueryParserService queryParserService, IndexFieldDataService fieldDataService) throws IOException {
XContentParser.Token token;
String fieldName = null;
CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester);

XContentParser contextParser = null;
CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptions = null;
CompletionSuggestionBuilder.RegexOptionsBuilder regexOptions = null;
Set<String> payloadFields = new HashSet<>(1);

while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
Expand All @@ -105,6 +105,8 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma
if (parser.booleanValue()) {
fuzzyOptions = new CompletionSuggestionBuilder.FuzzyOptionsBuilder();
}
} else if (token == XContentParser.Token.VALUE_STRING && "payload".equals(fieldName)) {
payloadFields.add(parser.text());
}
}
} else if (token == XContentParser.Token.START_OBJECT) {
Expand Down Expand Up @@ -158,6 +160,18 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma
} else {
throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("payload".equals(fieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_STRING) {
payloadFields.add(parser.text());
} else {
throw new IllegalArgumentException("suggester [completion] expected string values in [payload] array");
}
}
} else {
throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]");
}
} else {
throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]");
}
Expand All @@ -177,10 +191,13 @@ public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, Ma
contextParser.close();
}

suggestion.fieldType(type);
suggestion.setFieldType(type);
suggestion.setFuzzyOptionsBuilder(fuzzyOptions);
suggestion.setRegexOptionsBuilder(regexOptions);
suggestion.setQueryContexts(queryContexts);
suggestion.setMapperService(mapperService);
suggestion.setFieldData(fieldDataService);
suggestion.setPayloadFields(payloadFields);
// TODO: pass a query builder or the query itself?
// now we do it in CompletionSuggester#toQuery(CompletionSuggestionContext)
return suggestion;
Expand Down
38 changes: 31 additions & 7 deletions core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.elasticsearch.search.suggest.completion;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.CollectionTerminatedException;
import org.apache.lucene.search.IndexSearcher;
Expand All @@ -30,6 +31,9 @@
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.text.StringText;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestContextParser;
Expand All @@ -50,7 +54,7 @@ public SuggestContextParser getContextParser() {
@Override
protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> innerExecute(String name,
CompletionSuggestionContext suggestionContext, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
if (suggestionContext.fieldType() == null) {
if (suggestionContext.getFieldType() == null) {
throw new ElasticsearchException("Field [" + suggestionContext.getField() + "] is not a completion suggest field");
}
CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize());
Expand All @@ -67,18 +71,38 @@ protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Sugges
// if we index a suggestion with n contexts, the suggestion and all its contexts
// would count as n hits rather than 1, so we have to multiply the desired size
// with n to get a suggestion with all n contexts
final String key = suggestDoc.key.toString();
final float score = suggestDoc.score;
final Map.Entry<String, CharSequence> contextEntry;
if (suggestionContext.fieldType().hasContextMappings() && suggestDoc.context != null) {
contextEntry = suggestionContext.fieldType().getContextMappings().getNamedContext(suggestDoc.context);
if (suggestionContext.getFieldType().hasContextMappings() && suggestDoc.context != null) {
contextEntry = suggestionContext.getFieldType().getContextMappings().getNamedContext(suggestDoc.context);
} else {
assert suggestDoc.context == null;
contextEntry = null;
}
final Option value = results.get(suggestDoc.doc);
final CompletionSuggestion.Entry.Option value = results.get(suggestDoc.doc);
if (value == null) {
final Option option = new Option(suggestDoc.doc, new StringText(key), score, contextEntry);
final Map<String, List<Object>> payload;
Set<String> payloadFields = suggestionContext.getPayloadFields();
if (!payloadFields.isEmpty()) {
int readerIndex = ReaderUtil.subIndex(suggestDoc.doc, searcher.getIndexReader().leaves());
LeafReaderContext subReaderContext = searcher.getIndexReader().leaves().get(readerIndex);
int subDocId = suggestDoc.doc - subReaderContext.docBase;
payload = new LinkedHashMap<>(payloadFields.size());
for (String field : payloadFields) {
MappedFieldType fieldType = suggestionContext.getMapperService().smartNameFieldType(field);
if (fieldType != null) {
Copy link
Contributor

@mikemccand mikemccand Sep 18, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, if fieldType is null it means the use asked for a non-existent payloads field? Can we throw an exc (earlier) so user realizes this?

Copy link
Contributor Author

@areek areek Sep 18, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed to throw an exception when we encounter non-existent payload fields

AtomicFieldData data = suggestionContext.getFieldData().getForField(fieldType).load(subReaderContext);
ScriptDocValues scriptValues = data.getScriptValues();
scriptValues.setNextDocId(subDocId);
payload.put(field, scriptValues.getValues());
} else {
throw new ElasticsearchException("Payload field [" + field + "] does not exist");
}
}
} else {
payload = Collections.emptyMap();
}
final CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(new StringText(suggestDoc.key.toString()), suggestDoc.score, contextEntry, payload);
results.put(suggestDoc.doc, option);
} else {
value.addContextEntry(contextEntry);
Expand All @@ -100,7 +124,7 @@ protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Sugges
so the CompletionSuggestionContext will have only the query instead
*/
private CompletionQuery toQuery(CompletionSuggestionContext suggestionContext) throws IOException {
CompletionFieldMapper.CompletionFieldType fieldType = suggestionContext.fieldType();
CompletionFieldMapper.CompletionFieldType fieldType = suggestionContext.getFieldType();
final CompletionQuery query;
if (suggestionContext.getPrefix() != null) {
if (suggestionContext.getFuzzyOptionsBuilder() != null) {
Expand Down
Loading