diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionTokenStream.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionTokenStream.java index b0dfef28514ad..c3d53691aabe2 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionTokenStream.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionTokenStream.java @@ -27,7 +27,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.Transition; -import org.apache.lucene.util.fst.XUtil; +import org.apache.lucene.util.fst.Util; import java.io.IOException; import java.util.BitSet; @@ -107,7 +107,7 @@ public boolean incrementToken() throws IOException { * produced. Multi Fields have the same surface form and therefore sum up */ posInc = 0; - XUtil.toBytesRef(finiteStrings.next(), bytesAtt.builder()); // now we have UTF-8 + Util.toBytesRef(finiteStrings.next(), bytesAtt.builder()); // now we have UTF-8 if (charTermAttribute != null) { charTermAttribute.setLength(0); charTermAttribute.append(bytesAtt.toUTF16()); diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionWeight.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionWeight.java index 4fd7093bab4e4..2c6094a8a39b1 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionWeight.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/CompletionWeight.java @@ -27,7 +27,6 @@ import org.apache.lucene.util.automaton.Automaton; import java.io.IOException; -import java.util.List; import java.util.Set; /** @@ -35,7 +34,7 @@ * score and explain these queries. * * Subclasses can override {@link #setNextMatch(IntsRef)}, - * {@link #boost()} and {@link #contexts()} + * {@link #boost()} and {@link #context()} * to calculate the boost and extract the context of * a matched path prefix. * @@ -103,7 +102,7 @@ public BulkScorer bulkScorer(final LeafReaderContext context, Bits acceptDocs) t * Set for every partial path in the index that matched the query * automaton. * - * Subclasses should override {@link #boost()} and {@link #contexts()} + * Subclasses should override {@link #boost()} and {@link #context()} * to return an appropriate value with respect to the current pathPrefix. * * @param pathPrefix the prefix of a matched path @@ -125,7 +124,7 @@ protected float boost() { * * @return suggestion context */ - protected List contexts() { + protected CharSequence context() { return null; } diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextQuery.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextQuery.java index 4989b5856ff5f..5aa0e636c8b1f 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextQuery.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextQuery.java @@ -26,8 +26,7 @@ import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; -import org.apache.lucene.util.automaton.RegExp; -import org.apache.lucene.util.fst.XUtil; +import org.apache.lucene.util.fst.Util; import java.io.IOException; import java.util.*; @@ -56,16 +55,16 @@ * or {@link FuzzyCompletionQuery} query. * *
  • - * To suggest across all contexts with the same boost, - * use '*' as the context in {@link #addContext(CharSequence)})}. - * This can be combined with specific contexts with different boosts. + * To suggest across all contexts, use {@link #addAllContexts()}. + * When no context is added, the default behaviour is to suggest across + * all contexts. *
    • *
  • * To apply the same boost to multiple contexts sharing the same prefix, * Use {@link #addContext(CharSequence, float, boolean)} with the common * context prefix, boost and set exact to false. *
  • - * Using this query against a {@link SuggestField} (not context enabled), + * Using this query against a {@link org.apache.lucene.search.suggest.document.SuggestField} (not context enabled), * would yield results ignoring any context filtering/boosting *
    • * @@ -73,8 +72,9 @@ * @lucene.experimental */ public class ContextQuery extends CompletionQuery { - protected Map contexts; - protected boolean matchAllContexts = false; + private IntsRefBuilder scratch = new IntsRefBuilder(); + private Map contexts; + private boolean matchAllContexts = false; /** Inner completion query */ protected CompletionQuery innerQuery; @@ -85,14 +85,13 @@ public class ContextQuery extends CompletionQuery { * Use {@link #addContext(CharSequence, float, boolean)} * to add context(s) with boost */ - public ContextQuery(CompletionQuery innerQuery) { - super(innerQuery.getTerm(), innerQuery.getFilter()); - /* + public ContextQuery(CompletionQuery query) { + super(query.getTerm(), query.getFilter()); if (query instanceof ContextQuery) { throw new IllegalArgumentException("'query' parameter must not be of type " + this.getClass().getSimpleName()); - }*/ - this.innerQuery = innerQuery; + } + this.innerQuery = query; contexts = new HashMap<>(); } @@ -121,51 +120,31 @@ public void addContext(CharSequence context, float boost, boolean exact) { for (int i = 0; i < context.length(); i++) { if (ContextSuggestField.CONTEXT_SEPARATOR == context.charAt(i)) { throw new IllegalArgumentException("Illegal value [" + context + "] UTF-16 codepoint [0x" - + Integer.toHexString((int) context.charAt(i))+ "] at position " + i + " is a reserved character"); + + Integer.toHexString((int) context.charAt(i))+ "] at position " + i + " is a reserved character"); } } - contexts.put(context, new ContextMetaData(boost, exact)); + contexts.put(IntsRef.deepCopyOf(Util.toIntsRef(new BytesRef(context), scratch)), new ContextMetaData(boost, exact)); } + /** + * Add all contexts with a boost of 1f + */ public void addAllContexts() { matchAllContexts = true; } - protected Automaton contextAutomaton() { - final Automaton matchAllAutomaton = new RegExp(".*").toAutomaton(); - final Automaton sep = Automata.makeChar(ContextSuggestField.CONTEXT_SEPARATOR); - if (matchAllContexts || contexts.size() == 0) { - return Operations.concatenate(matchAllAutomaton, sep); - } else { - Automaton contextsAutomaton = null; - for (Map.Entry entry : contexts.entrySet()) { - final ContextMetaData contextMetaData = entry.getValue(); - Automaton contextAutomaton = Automata.makeString(entry.getKey().toString()); - if (contextMetaData.exact == false) { - contextAutomaton = Operations.concatenate(contextAutomaton, matchAllAutomaton); - } - contextAutomaton = Operations.concatenate(contextAutomaton, sep); - if (contextsAutomaton == null) { - contextsAutomaton = contextAutomaton; - } else { - contextsAutomaton = Operations.union(contextsAutomaton, contextAutomaton); - } - } - return contextsAutomaton; - } - } - @Override public String toString(String field) { StringBuilder buffer = new StringBuilder(); - for (CharSequence context : contexts.keySet()) { + BytesRefBuilder scratch = new BytesRefBuilder(); + for (IntsRef context : contexts.keySet()) { if (buffer.length() != 0) { buffer.append(","); } else { buffer.append("contexts"); buffer.append(":["); } - buffer.append(context); + buffer.append(Util.toBytesRef(context, scratch).utf8ToString()); ContextMetaData metaData = contexts.get(context); if (metaData.exact == false) { buffer.append("*"); @@ -185,27 +164,20 @@ public String toString(String field) { @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { final CompletionWeight innerWeight = ((CompletionWeight) innerQuery.createWeight(searcher, needsScores)); - Automaton contextsAutomaton; - if (innerQuery instanceof ContextQuery) { - contextsAutomaton = Operations.concatenate(contextAutomaton(), innerWeight.getAutomaton()); - } else { - // if separators are preserved the fst contains a SEP_LABEL - // behind each gap. To have a matching automaton, we need to - // include the SEP_LABEL in the query as well - Automaton optionalSepLabel = Operations.optional(Automata.makeChar(CompletionAnalyzer.SEP_LABEL)); - Automaton prefixAutomaton = Operations.concatenate(optionalSepLabel, innerWeight.getAutomaton()); - contextsAutomaton = Operations.concatenate(contextAutomaton(), prefixAutomaton); - } + // if separators are preserved the fst contains a SEP_LABEL + // behind each gap. To have a matching automaton, we need to + // include the SEP_LABEL in the query as well + Automaton optionalSepLabel = Operations.optional(Automata.makeChar(CompletionAnalyzer.SEP_LABEL)); + Automaton prefixAutomaton = Operations.concatenate(optionalSepLabel, innerWeight.getAutomaton()); + Automaton contextsAutomaton = Operations.concatenate(toContextAutomaton(contexts, matchAllContexts), prefixAutomaton); contextsAutomaton = Operations.determinize(contextsAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); final Map contextMap = new HashMap<>(contexts.size()); final TreeSet contextLengths = new TreeSet<>(); - IntsRefBuilder scratch = new IntsRefBuilder(); - for (Map.Entry entry : contexts.entrySet()) { - BytesRef ref = new BytesRef(entry.getKey()); + for (Map.Entry entry : contexts.entrySet()) { ContextMetaData contextMetaData = entry.getValue(); - contextMap.put(IntsRef.deepCopyOf(XUtil.toIntsRef(ref, scratch)), contextMetaData.boost); - contextLengths.add(scratch.length()); + contextMap.put(entry.getKey(), contextMetaData.boost); + contextLengths.add(entry.getKey().length); } int[] contextLengthArray = new int[contextLengths.size()]; final Iterator iterator = contextLengths.descendingIterator(); @@ -215,8 +187,47 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I return new ContextCompletionWeight(this, contextsAutomaton, innerWeight, contextMap, contextLengthArray); } + private static Automaton toContextAutomaton(final Map contexts, final boolean matchAllContexts) { + final Automaton matchAllAutomaton = Operations.repeat(Automata.makeAnyString()); + final Automaton sep = Automata.makeChar(ContextSuggestField.CONTEXT_SEPARATOR); + if (matchAllContexts || contexts.size() == 0) { + return Operations.concatenate(matchAllAutomaton, sep); + } else { + Automaton contextsAutomaton = null; + for (Map.Entry entry : contexts.entrySet()) { + final ContextMetaData contextMetaData = entry.getValue(); + final IntsRef ref = entry.getKey(); + Automaton contextAutomaton = Automata.makeString(ref.ints, ref.offset, ref.length); + if (contextMetaData.exact == false) { + contextAutomaton = Operations.concatenate(contextAutomaton, matchAllAutomaton); + } + contextAutomaton = Operations.concatenate(contextAutomaton, sep); + if (contextsAutomaton == null) { + contextsAutomaton = contextAutomaton; + } else { + contextsAutomaton = Operations.union(contextsAutomaton, contextAutomaton); + } + } + return contextsAutomaton; + } + } + + /** + * Holder for context value meta data + */ private static class ContextMetaData { + + /** + * Boost associated with a + * context value + */ private final float boost; + + /** + * flag to indicate whether the context + * value should be treated as an exact + * value or a context prefix + */ private final boolean exact; private ContextMetaData(float boost, boolean exact) { @@ -274,8 +285,7 @@ private void setInnerWeight(IntsRef ref, int offset) { if (ref.ints[ref.offset + i] == ContextSuggestField.CONTEXT_SEPARATOR) { if (i > 0) { refBuilder.copyInts(ref.ints, ref.offset, i); - currentContext = XUtil.toBytesRef(refBuilder.get(), scratch).utf8ToString(); - refBuilder.clear(); + currentContext = Util.toBytesRef(refBuilder.get(), scratch).utf8ToString(); } else { currentContext = null; } @@ -294,13 +304,8 @@ private void setInnerWeight(IntsRef ref, int offset) { } @Override - protected List contexts() { - final List contexts = new ArrayList<>(); - contexts.add(currentContext); - if (innerWeight instanceof ContextCompletionWeight) { - contexts.addAll(innerWeight.contexts()); - } - return contexts; + protected CharSequence context() { + return currentContext; } @Override diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextSuggestField.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextSuggestField.java index 4975b79768f13..bcef26f308009 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextSuggestField.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/ContextSuggestField.java @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import java.io.IOException; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.Set; @@ -70,31 +71,31 @@ public ContextSuggestField(String name, String value, int weight, CharSequence.. validate(value); this.contexts = new HashSet<>((contexts != null) ? contexts.length : 0); if (contexts != null) { - for (CharSequence context : contexts) { - validate(context); - this.contexts.add(context); - } + Collections.addAll(this.contexts, contexts); } } /** - * Sub-classes can inject contexts at - * index-time by overriding + * Expert: Sub-classes can inject contexts at + * index-time */ - protected Set contexts() { + protected Iterable contexts() { return contexts; } @Override protected CompletionTokenStream wrapTokenStream(TokenStream stream) { - CompletionTokenStream completionTokenStream; + for (CharSequence context : contexts()) { + validate(context); + } PrefixTokenFilter prefixTokenFilter = new PrefixTokenFilter(stream, (char) CONTEXT_SEPARATOR, contexts()); + CompletionTokenStream completionTokenStream; if (stream instanceof CompletionTokenStream) { completionTokenStream = (CompletionTokenStream) stream; completionTokenStream = new CompletionTokenStream(prefixTokenFilter, - completionTokenStream.preserveSep, - completionTokenStream.preservePositionIncrements, - completionTokenStream.maxGraphExpansions); + completionTokenStream.preserveSep, + completionTokenStream.preservePositionIncrements, + completionTokenStream.maxGraphExpansions); } else { completionTokenStream = new CompletionTokenStream(prefixTokenFilter); } @@ -161,11 +162,11 @@ public void reset() throws IOException { } } - protected void validate(final CharSequence value) { + private void validate(final CharSequence value) { for (int i = 0; i < value.length(); i++) { if (CONTEXT_SEPARATOR == value.charAt(i)) { throw new IllegalArgumentException("Illegal value [" + value + "] UTF-16 codepoint [0x" - + Integer.toHexString((int) value.charAt(i))+ "] at position " + i + " is a reserved character"); + + Integer.toHexString((int) value.charAt(i))+ "] at position " + i + " is a reserved character"); } } } diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/NRTSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/NRTSuggester.java index 59a2681291c3c..ef1437447e863 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/NRTSuggester.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/NRTSuggester.java @@ -123,7 +123,7 @@ public Collection getChildResources() { * the matched partial paths. Upon reaching a completed path, {@link CompletionScorer#accept(int)} * and {@link CompletionScorer#score(float, float)} is used on the document id, index weight * and query boost to filter and score the entry, before being collected via - * {@link TopSuggestDocsCollector#collect(int, CharSequence, CharSequence[], float)} + * {@link TopSuggestDocsCollector#collect(int, CharSequence, CharSequence, float)} */ public void lookup(final CompletionScorer scorer, final TopSuggestDocsCollector collector) throws IOException { final double liveDocsRatio = calculateLiveDocRatio(scorer.reader.numDocs(), scorer.reader.maxDoc()); @@ -148,7 +148,7 @@ protected boolean acceptResult(XUtil.FSTPath> path) { } try { float score = scorer.score(decode(path.cost.output1), path.boost); - collector.collect(docID, spare.toCharsRef(), path.contexts, score); + collector.collect(docID, spare.toCharsRef(), path.context, score); return true; } catch (IOException e) { throw new RuntimeException(e); @@ -158,14 +158,7 @@ protected boolean acceptResult(XUtil.FSTPath> path) { for (FSTUtil.Path> path : prefixPaths) { scorer.weight.setNextMatch(path.input.get()); - List contexts = scorer.weight.contexts(); - final CharSequence[] contextArray; - if (contexts != null) { - contextArray = contexts.toArray(new CharSequence[contexts.size()]); - } else { - contextArray = null; - } - searcher.addStartPaths(path.fstNode, path.output, false, path.input, scorer.weight.boost(), contextArray); + searcher.addStartPaths(path.fstNode, path.output, false, path.input, scorer.weight.boost(), scorer.weight.context()); } // hits are also returned by search() // we do not use it, instead collect at acceptResult diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/SuggestScoreDocPriorityQueue.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/SuggestScoreDocPriorityQueue.java index 7a0bc5b398e40..745c029fd841c 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/SuggestScoreDocPriorityQueue.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/SuggestScoreDocPriorityQueue.java @@ -35,8 +35,13 @@ public SuggestScoreDocPriorityQueue(int size) { @Override protected boolean lessThan(TopSuggestDocs.SuggestScoreDoc a, TopSuggestDocs.SuggestScoreDoc b) { if (a.score == b.score) { - // prefer smaller doc id, in case of a tie - return a.doc > b.doc; + int cmp = a.compareTo(b); + if (cmp == 0) { + // prefer smaller doc id, in case of a tie + return a.doc > b.doc; + } else { + return cmp < 0; + } } return a.score < b.score; } diff --git a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/TopSuggestDocs.java b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/TopSuggestDocs.java index 05f8e20373923..003d82826dd48 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/xdocument/TopSuggestDocs.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/xdocument/TopSuggestDocs.java @@ -48,7 +48,7 @@ public static class SuggestScoreDoc extends ScoreDoc implements Comparable { public T cost; public final IntsRefBuilder input; public final float boost; - public final CharSequence[] contexts; + public final CharSequence context; /** Sole constructor */ public FSTPath(T cost, FST.Arc arc, IntsRefBuilder input) { this(cost, arc, input, 0, null); } - public FSTPath(T cost, FST.Arc arc, IntsRefBuilder input, float boost, CharSequence[] contexts) { + public FSTPath(T cost, FST.Arc arc, IntsRefBuilder input, float boost, CharSequence context) { this.arc = new FST.Arc().copyFrom(arc); this.cost = cost; this.input = input; this.boost = boost; - this.contexts = contexts; + this.context = context; } public FSTPath newPath(T cost, IntsRefBuilder input) { - return new FSTPath<>(cost, this.arc, input, this.boost, this.contexts); + return new FSTPath<>(cost, this.arc, input, this.boost, this.context); } @Override public String toString() { - return "input=" + input.get() + " cost=" + cost + "context=" + contexts + "boost=" + boost; + return "input=" + input.get() + " cost=" + cost + "context=" + context + "boost=" + boost; } } @@ -387,14 +387,14 @@ public void addStartPaths(FST.Arc node, T startOutput, boolean allowEmptyStri /** Adds all leaving arcs, including 'finished' arc, if * the node is final, from this node into the queue. */ public void addStartPaths(FST.Arc node, T startOutput, boolean allowEmptyString, IntsRefBuilder input, - float boost, CharSequence[] contexts) throws IOException { + float boost, CharSequence context) throws IOException { // De-dup NO_OUTPUT since it must be a singleton: if (startOutput.equals(fst.outputs.getNoOutput())) { startOutput = fst.outputs.getNoOutput(); } - FSTPath path = new FSTPath<>(startOutput, node, input, boost, contexts); + FSTPath path = new FSTPath<>(startOutput, node, input, boost, context); fst.readFirstTargetArc(node, path.arc, bytesReader); //System.out.println("add start paths"); diff --git a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index b8f1276d23d3f..a8ba3c8109859 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -25,9 +25,9 @@ import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; /** @@ -58,11 +58,13 @@ public PostingsFormat getPostingsFormatForField(String field) { final MappedFieldType indexName = mapperService.indexName(field); if (indexName == null) { logger.warn("no index mapper found for field: [{}] returning default postings format", field); - } else if (indexName instanceof CompletionFieldMapper.CompletionFieldType) { + } else if (indexName instanceof OldCompletionFieldMapper.CompletionFieldType) { // CompletionFieldMapper needs a special postings format - final CompletionFieldMapper.CompletionFieldType fieldType = (CompletionFieldMapper.CompletionFieldType) indexName; + final OldCompletionFieldMapper.CompletionFieldType fieldType = (OldCompletionFieldMapper.CompletionFieldType) indexName; final PostingsFormat defaultFormat = super.getPostingsFormatForField(field); return fieldType.postingsFormat(defaultFormat); + } else if (indexName instanceof CompletionFieldMapper.CompletionFieldType) { + return CompletionFieldMapper.CompletionFieldType.postingsFormat(); } return super.getPostingsFormatForField(field); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java index 73c92ded424c7..d800eeafe0648 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java @@ -23,7 +23,6 @@ import org.elasticsearch.index.mapper.core.*; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.internal.*; import org.elasticsearch.index.mapper.ip.IpFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; @@ -104,6 +103,10 @@ public static GeoShapeFieldMapper.Builder geoShapeField(String name) { return new GeoShapeFieldMapper.Builder(name); } + public static OldCompletionFieldMapper.Builder oldCompletionField(String name) { + return new OldCompletionFieldMapper.Builder(name); + } + public static CompletionFieldMapper.Builder completionField(String name) { return new CompletionFieldMapper.Builder(name); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java index 6297c6a2a7842..07983f4351833 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java @@ -18,151 +18,100 @@ */ package org.elasticsearch.index.mapper.core; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.Field; -import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.search.suggest.xdocument.*; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.NumberType; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperException; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MergeMappingException; -import org.elasticsearch.index.mapper.MergeResult; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.search.suggest.completion.AnalyzingCompletionLookupProvider; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream; -import org.elasticsearch.search.suggest.context.ContextBuilder; -import org.elasticsearch.search.suggest.context.ContextMapping; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextConfig; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; +import org.elasticsearch.search.suggest.completion.CompletionSuggester; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; +import org.elasticsearch.search.suggest.completion.context.ContextMappingsParser; import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; - -import static org.elasticsearch.index.mapper.MapperBuilders.completionField; +import java.util.*; + +import static org.elasticsearch.index.mapper.MapperBuilders.*; import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; +import static org.elasticsearch.search.suggest.completion.context.ContextMappingsParser.parseContext; /** + * Mapper for completion field. The field values are indexed as a weighted FST for + * fast auto-completion/search-as-you-type functionality.
    * + * Type properties:
    + *
      + *
    • "analyzer": "simple", (default)
      • + *
    • "search_analyzer": "simple", (default)
      • + *
    • "preserve_separators" : true, (default)
      • + *
    • "preserve_position_increments" : true (default)
      • + *
    • "min_input_length": 50 (default)
      • + *
    • "contexts" : CONTEXTS
      • + *
    + * see {@link ContextMappings#load(Object, Version)} for CONTEXTS
    + * see {@link #parse(ParseContext)} for acceptable inputs for indexing
    + *

    + * This field type constructs completion queries that are run + * against the weighted FST index by the {@link CompletionSuggester}. + * This field can also be extended to add search criteria to suggestions + * for query-time filtering and boosting (see {@link ContextMappings} */ -public class CompletionFieldMapper extends FieldMapper { +public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapperParser { public static final String CONTENT_TYPE = "completion"; public static class Defaults { - public static final CompletionFieldType FIELD_TYPE = new CompletionFieldType(); - + public static final MappedFieldType FIELD_TYPE = new CompletionFieldType(); static { FIELD_TYPE.setOmitNorms(true); FIELD_TYPE.freeze(); } - public static final boolean DEFAULT_PRESERVE_SEPARATORS = true; public static final boolean DEFAULT_POSITION_INCREMENTS = true; - public static final boolean DEFAULT_HAS_PAYLOADS = false; public static final int DEFAULT_MAX_INPUT_LENGTH = 50; } public static class Fields { // Mapping field names - public static final String ANALYZER = "analyzer"; + public static final ParseField ANALYZER = new ParseField("analyzer"); public static final ParseField SEARCH_ANALYZER = new ParseField("search_analyzer"); public static final ParseField PRESERVE_SEPARATORS = new ParseField("preserve_separators"); public static final ParseField PRESERVE_POSITION_INCREMENTS = new ParseField("preserve_position_increments"); - public static final String PAYLOADS = "payloads"; - public static final String TYPE = "type"; + public static final ParseField TYPE = new ParseField("type"); + public static final ParseField CONTEXTS = new ParseField("contexts"); public static final ParseField MAX_INPUT_LENGTH = new ParseField("max_input_length", "max_input_len"); // Content field names public static final String CONTENT_FIELD_NAME_INPUT = "input"; - public static final String CONTENT_FIELD_NAME_OUTPUT = "output"; - public static final String CONTENT_FIELD_NAME_PAYLOAD = "payload"; public static final String CONTENT_FIELD_NAME_WEIGHT = "weight"; - public static final String CONTEXT = "context"; + public static final String CONTENT_FIELD_NAME_CONTEXTS = "contexts"; } public static final Set ALLOWED_CONTENT_FIELD_NAMES = Sets.newHashSet(Fields.CONTENT_FIELD_NAME_INPUT, - Fields.CONTENT_FIELD_NAME_OUTPUT, Fields.CONTENT_FIELD_NAME_PAYLOAD, Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTEXT); - - public static class Builder extends FieldMapper.Builder { - - private boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS; - private boolean payloads = Defaults.DEFAULT_HAS_PAYLOADS; - private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; - private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH; - private SortedMap contextMapping = ContextMapping.EMPTY_MAPPING; - - public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - builder = this; - } - - public Builder payloads(boolean payloads) { - this.payloads = payloads; - return this; - } - - public Builder preserveSeparators(boolean preserveSeparators) { - this.preserveSeparators = preserveSeparators; - return this; - } - - public Builder preservePositionIncrements(boolean preservePositionIncrements) { - this.preservePositionIncrements = preservePositionIncrements; - return this; - } - - public Builder maxInputLength(int maxInputLength) { - if (maxInputLength <= 0) { - throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() + " must be > 0 but was [" + maxInputLength + "]"); - } - this.maxInputLength = maxInputLength; - return this; - } - - public Builder contextMapping(SortedMap contextMapping) { - this.contextMapping = contextMapping; - return this; - } - - @Override - public CompletionFieldMapper build(Mapper.BuilderContext context) { - setupFieldType(context); - CompletionFieldType completionFieldType = (CompletionFieldType)fieldType; - completionFieldType.setProvider(new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, payloads)); - completionFieldType.setContextMapping(contextMapping); - return new CompletionFieldMapper(name, fieldType, maxInputLength, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); - } - - } + Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTENT_FIELD_NAME_CONTEXTS); public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + return new OldCompletionFieldMapper.TypeParser().parse(name, node, parserContext); + } CompletionFieldMapper.Builder builder = completionField(name); NamedAnalyzer indexAnalyzer = null; NamedAnalyzer searchAnalyzer = null; + boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; + boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS; for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String fieldName = entry.getKey(); @@ -170,34 +119,29 @@ public static class TypeParser implements Mapper.TypeParser { if (fieldName.equals("type")) { continue; } - if (Fields.ANALYZER.equals(fieldName) || // index_analyzer is for backcompat, remove for v3.0 - fieldName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - + if (parserContext.parseFieldMatcher().match(fieldName, Fields.ANALYZER)) { indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); iterator.remove(); } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.SEARCH_ANALYZER)) { searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); iterator.remove(); - } else if (fieldName.equals(Fields.PAYLOADS)) { - builder.payloads(Boolean.parseBoolean(fieldNode.toString())); - iterator.remove(); } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_SEPARATORS)) { - builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString())); + preserveSeparators = Boolean.parseBoolean(fieldNode.toString()); iterator.remove(); } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_POSITION_INCREMENTS)) { - builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString())); + preservePositionIncrements = Boolean.parseBoolean(fieldNode.toString()); iterator.remove(); } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.MAX_INPUT_LENGTH)) { builder.maxInputLength(Integer.parseInt(fieldNode.toString())); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) { + } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.CONTEXTS)) { + builder.contextMappings(ContextMappings.load(fieldNode, parserContext.indexVersionCreated())); iterator.remove(); - } else if (fieldName.equals(Fields.CONTEXT)) { - builder.contextMapping(ContextBuilder.loadMappings(fieldNode, parserContext.indexVersionCreated())); + } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) { iterator.remove(); } } - + if (indexAnalyzer == null) { if (searchAnalyzer != null) { throw new MapperParsingException("analyzer on completion field [" + name + "] must be set when search_analyzer is set"); @@ -206,9 +150,11 @@ public static class TypeParser implements Mapper.TypeParser { } else if (searchAnalyzer == null) { searchAnalyzer = indexAnalyzer; } - builder.indexAnalyzer(indexAnalyzer); - builder.searchAnalyzer(searchAnalyzer); - + + CompletionAnalyzer completionIndexAnalyzer = new CompletionAnalyzer(indexAnalyzer, preserveSeparators, preservePositionIncrements); + CompletionAnalyzer completionSearchAnalyzer = new CompletionAnalyzer(searchAnalyzer, preserveSeparators, preservePositionIncrements); + builder.indexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), indexAnalyzer.scope(), completionIndexAnalyzer)); + builder.searchAnalyzer(new NamedAnalyzer(searchAnalyzer.name(), searchAnalyzer.scope(), completionSearchAnalyzer)); return builder; } @@ -222,19 +168,72 @@ private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) } public static final class CompletionFieldType extends MappedFieldType { - private PostingsFormat postingsFormat; - private AnalyzingCompletionLookupProvider analyzingSuggestLookupProvider; - private SortedMap contextMapping = ContextMapping.EMPTY_MAPPING; + + private static PostingsFormat postingsFormat; + private ContextMappings contextMappings = null; public CompletionFieldType() { setFieldDataType(null); } - protected CompletionFieldType(CompletionFieldType ref) { + private CompletionFieldType(CompletionFieldType ref) { super(ref); - this.postingsFormat = ref.postingsFormat; - this.analyzingSuggestLookupProvider = ref.analyzingSuggestLookupProvider; - this.contextMapping = ref.contextMapping; + this.contextMappings = ref.contextMappings; + } + + private void setContextMappings(ContextMappings contextMappings) { + checkIfFrozen(); + this.contextMappings = contextMappings; + } + + /** + * @return true if there are one or more context mappings defined + * for this field type + */ + public boolean hasContextMappings() { + return contextMappings != null; + } + + /** + * @return associated context mappings for this field type + */ + public ContextMappings getContextMappings() { + return contextMappings; + } + + /** + * @return postings format to use for this field-type + */ + public static synchronized PostingsFormat postingsFormat() { + if (postingsFormat == null) { + postingsFormat = new Completion50PostingsFormat(); + } + return postingsFormat; + } + + /** + * Completion prefix query + */ + public CompletionQuery prefixQuery(Object value) { + return new PrefixCompletionQuery(searchAnalyzer().analyzer(), createTerm(value)); + } + + /** + * Completion prefix regular expression query + */ + public CompletionQuery regexpQuery(Object value, int flags, int maxDeterminizedStates) { + return new RegexCompletionQuery(createTerm(value), flags, maxDeterminizedStates); + } + + /** + * Completion prefix fuzzy query + */ + public CompletionQuery fuzzyQuery(String value, Fuzziness fuzziness, int nonFuzzyPrefixLength, + int minFuzzyPrefixLength, int maxExpansions, boolean transpositions, + boolean unicodeAware) { + return new FuzzyCompletionQuery(searchAnalyzer().analyzer(), createTerm(value), null, + fuzziness.asDistance(), transpositions, nonFuzzyPrefixLength, minFuzzyPrefixLength, + unicodeAware, maxExpansions); } @Override @@ -251,48 +250,20 @@ public String typeName() { public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) { super.checkCompatibility(fieldType, conflicts, strict); CompletionFieldType other = (CompletionFieldType)fieldType; - if (analyzingSuggestLookupProvider.hasPayloads() != other.analyzingSuggestLookupProvider.hasPayloads()) { - conflicts.add("mapper [" + names().fullName() + "] has different payload values"); - } - if (analyzingSuggestLookupProvider.getPreservePositionsIncrements() != other.analyzingSuggestLookupProvider.getPreservePositionsIncrements()) { + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer().analyzer(); + CompletionAnalyzer otherAnalyzer = (CompletionAnalyzer) other.indexAnalyzer().analyzer(); + + if (analyzer.preservePositionIncrements() != otherAnalyzer.preservePositionIncrements()) { conflicts.add("mapper [" + names().fullName() + "] has different 'preserve_position_increments' values"); } - if (analyzingSuggestLookupProvider.getPreserveSep() != other.analyzingSuggestLookupProvider.getPreserveSep()) { + if (analyzer.preserveSep() != otherAnalyzer.preserveSep()) { conflicts.add("mapper [" + names().fullName() + "] has different 'preserve_separators' values"); } - if(!ContextMapping.mappingsAreEqual(getContextMapping(), other.getContextMapping())) { - conflicts.add("mapper [" + names().fullName() + "] has different 'context_mapping' values"); - } - } - - public void setProvider(AnalyzingCompletionLookupProvider provider) { - checkIfFrozen(); - this.analyzingSuggestLookupProvider = provider; - } - - public synchronized PostingsFormat postingsFormat(PostingsFormat in) { - if (in instanceof Completion090PostingsFormat) { - throw new IllegalStateException("Double wrapping of " + Completion090PostingsFormat.class); - } - if (postingsFormat == null) { - postingsFormat = new Completion090PostingsFormat(in, analyzingSuggestLookupProvider); + if (hasContextMappings() != other.hasContextMappings()) { + conflicts.add("mapper [" + names().fullName() + "] has different context mapping"); + } else if (hasContextMappings() && contextMappings.equals(other.contextMappings) == false) { + conflicts.add("mapper [" + names().fullName() + "] has different 'context_mappings' values"); } - return postingsFormat; - } - - public void setContextMapping(SortedMap contextMapping) { - checkIfFrozen(); - this.contextMapping = contextMapping; - } - - /** Get the context mapping associated with this completion field */ - public SortedMap getContextMapping() { - return contextMapping; - } - - /** @return true if a context mapping has been defined */ - public boolean requiresContext() { - return contextMapping.isEmpty() == false; } @Override @@ -307,13 +278,58 @@ public String value(Object value) { public boolean isSortable() { return false; } + } - private static final BytesRef EMPTY = new BytesRef(); + /** + * Builder for {@link CompletionFieldMapper} + */ + public static class Builder extends FieldMapper.Builder { + + private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH; + private ContextMappings contextMappings = null; + + /** + * @param name of the completion field to build + */ + public Builder(String name) { + super(name, new CompletionFieldType()); + builder = this; + } + + /** + * @param maxInputLength maximum expected prefix length + * NOTE: prefixes longer than this will + * be truncated + */ + public Builder maxInputLength(int maxInputLength) { + if (maxInputLength <= 0) { + throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() + " must be > 0 but was [" + maxInputLength + "]"); + } + this.maxInputLength = maxInputLength; + return this; + } + + /** + * Add context mapping to this field + * @param contextMappings see {@link ContextMappings#load(Object, Version)} + */ + public Builder contextMappings(ContextMappings contextMappings) { + this.contextMappings = contextMappings; + return this; + } + + @Override + public CompletionFieldMapper build(BuilderContext context) { + setupFieldType(context); + ((CompletionFieldType) fieldType).setContextMappings(contextMappings); + return new CompletionFieldMapper(name, fieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, maxInputLength); + } + } private int maxInputLength; - public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, int maxInputLength, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, int maxInputLength) { super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); this.maxInputLength = maxInputLength; } @@ -323,216 +339,152 @@ public CompletionFieldType fieldType() { return (CompletionFieldType) super.fieldType(); } + /** + * Parses and indexes inputs + * + * Parsing: + * Acceptable format: + * "STRING" - interpreted as field value (input) + * "ARRAY" - each element can be one of {@link #parse(ParseContext, Token, XContentParser, CompletionInputs)} + * "OBJECT" - see {@link #parse(ParseContext, Token, XContentParser, CompletionInputs)} + * + * Indexing: + * if context mappings are defined, delegates to {@link ContextMappings#addFields(ParseContext.Document, String, String, int, Map)} + * else adds inputs as a {@link org.apache.lucene.search.suggest.xdocument.SuggestField} + */ @Override public Mapper parse(ParseContext context) throws IOException { + // parse XContentParser parser = context.parser(); - XContentParser.Token token = parser.currentToken(); - if (token == XContentParser.Token.VALUE_NULL) { + Token token = parser.currentToken(); + CompletionInputs completionInputs = new CompletionInputs(); + if (token == Token.VALUE_NULL) { throw new MapperParsingException("completion field [" + fieldType().names().fullName() + "] does not support null values"); + } else if (token == Token.VALUE_STRING) { + completionInputs.add(parser.text(), 1, Collections.>emptyMap()); + } else if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { + parse(context, token, parser, completionInputs); + } + } else { + parse(context, token, parser, completionInputs); } - String surfaceForm = null; - BytesRef payload = null; - long weight = -1; - List inputs = Lists.newArrayListWithExpectedSize(4); - - SortedMap contextConfig = null; + // index + for (Map.Entry completionInput : completionInputs) { + String input = completionInput.getKey(); + if (input.length() > maxInputLength) { + final int len = correctSubStringLen(input, Math.min(maxInputLength, input.length())); + input = input.substring(0, len); + } + CompletionInputs.CompletionInputMetaData metaData = completionInput.getValue(); + if (fieldType().hasContextMappings()) { + fieldType().getContextMappings().addFields(context.doc(), fieldType().names().indexName(), + input, metaData.weight, metaData.contexts); + } else { + context.doc().add(new org.apache.lucene.search.suggest.xdocument.SuggestField(fieldType().names().indexName(), input, metaData.weight)); + } + } + multiFields.parse(this, context); + return null; + } - if (token == XContentParser.Token.VALUE_STRING) { + /** + * Acceptable inputs: + * "STRING" - interpreted as the field value (input) + * "OBJECT" - { "input": STRING|ARRAY, "weight": STRING|INT, "contexts": ARRAY|OBJECT } + * + * NOTE: for "contexts" parsing see {@link ContextMappings#parseContext(ParseContext, XContentParser)} + */ + private void parse(ParseContext parseContext, Token token, XContentParser parser, CompletionInputs completionInputs) throws IOException { + Set inputs = new HashSet<>(); + Map> contextsMap = new HashMap<>(); + int weight = 0; + String currentFieldName = null; + ContextMappings contextMappings = fieldType().getContextMappings(); + if (token == Token.VALUE_STRING) { inputs.add(parser.text()); - multiFields.parse(this, context); - } else { - String currentFieldName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { + } else if (token == Token.START_OBJECT) { + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == Token.FIELD_NAME) { currentFieldName = parser.currentName(); if (!ALLOWED_CONTENT_FIELD_NAMES.contains(currentFieldName)) { throw new IllegalArgumentException("Unknown field name[" + currentFieldName + "], must be one of " + ALLOWED_CONTENT_FIELD_NAMES); } - } else if (Fields.CONTEXT.equals(currentFieldName)) { - SortedMap configs = Maps.newTreeMap(); - - if (token == Token.START_OBJECT) { - while ((token = parser.nextToken()) != Token.END_OBJECT) { - String name = parser.text(); - ContextMapping mapping = fieldType().getContextMapping().get(name); - if (mapping == null) { - throw new ElasticsearchParseException("context [{}] is not defined", name); - } else { - token = parser.nextToken(); - configs.put(name, mapping.parseContext(context, parser)); - } - } - contextConfig = Maps.newTreeMap(); - for (ContextMapping mapping : fieldType().getContextMapping().values()) { - ContextConfig config = configs.get(mapping.name()); - contextConfig.put(mapping.name(), config==null ? mapping.defaultConfig() : config); - } - } else { - throw new ElasticsearchParseException("context must be an object"); - } - } else if (Fields.CONTENT_FIELD_NAME_PAYLOAD.equals(currentFieldName)) { - if (!isStoringPayloads()) { - throw new MapperException("Payloads disabled in mapping"); - } - if (token == XContentParser.Token.START_OBJECT) { - XContentBuilder payloadBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); - payload = payloadBuilder.bytes().toBytesRef(); - payloadBuilder.close(); - } else if (token.isValue()) { - payload = parser.utf8BytesOrNull(); - } else { - throw new MapperException("payload doesn't support type " + token); - } - } else if (token == XContentParser.Token.VALUE_STRING) { - if (Fields.CONTENT_FIELD_NAME_OUTPUT.equals(currentFieldName)) { - surfaceForm = parser.text(); - } + } else if (token == Token.VALUE_STRING) { if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { inputs.add(parser.text()); - } - if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { + } else if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { Number weightValue; try { weightValue = Long.parseLong(parser.text()); } catch (NumberFormatException e) { throw new IllegalArgumentException("Weight must be a string representing a numeric value, but was [" + parser.text() + "]"); } - weight = weightValue.longValue(); // always parse a long to make sure we don't get overflow - checkWeight(weight); + checkWeight(weightValue.longValue());// always parse a long to make sure we don't get overflow + weight = weightValue.intValue(); } - } else if (token == XContentParser.Token.VALUE_NUMBER) { + } else if (token == Token.VALUE_NUMBER) { if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { NumberType numberType = parser.numberType(); if (NumberType.LONG != numberType && NumberType.INT != numberType) { throw new IllegalArgumentException("Weight must be an integer, but was [" + parser.numberValue() + "]"); } - weight = parser.longValue(); // always parse a long to make sure we don't get overflow - checkWeight(weight); + checkWeight(parser.longValue()); // always parse a long to make sure we don't get overflow + weight = parser.intValue(); } - } else if (token == XContentParser.Token.START_ARRAY) { + } else if (token == Token.START_ARRAY) { if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { inputs.add(parser.text()); } + } else if (Fields.CONTENT_FIELD_NAME_CONTEXTS.equals(currentFieldName)) { + if (fieldType().hasContextMappings() == false) { + throw new IllegalArgumentException("Supplied context(s) to a non context enabled field: [" + fieldType().names().fullName() + "]"); + } + addContexts(contextsMap, parseContext(contextMappings, parseContext, parser)); + } + } else if (token == Token.START_OBJECT) { + if (Fields.CONTENT_FIELD_NAME_CONTEXTS.equals(currentFieldName)) { + if (fieldType().hasContextMappings() == false) { + throw new IllegalArgumentException("Supplied context(s) to a non context enabled field: [" + fieldType().names().fullName() + "]"); + } + addContexts(contextsMap, parseContext(contextMappings, parseContext, parser)); } } } - } - - if(contextConfig == null) { - contextConfig = Maps.newTreeMap(); - for (ContextMapping mapping : fieldType().getContextMapping().values()) { - contextConfig.put(mapping.name(), mapping.defaultConfig()); - } - } - - final ContextMapping.Context ctx = new ContextMapping.Context(contextConfig, context.doc()); - - payload = payload == null ? EMPTY : payload; - if (surfaceForm == null) { // no surface form use the input - for (String input : inputs) { - if (input.length() == 0) { - continue; - } - BytesRef suggestPayload = fieldType().analyzingSuggestLookupProvider.buildPayload(new BytesRef( - input), weight, payload); - context.doc().add(getCompletionField(ctx, input, suggestPayload)); - } } else { - BytesRef suggestPayload = fieldType().analyzingSuggestLookupProvider.buildPayload(new BytesRef( - surfaceForm), weight, payload); - for (String input : inputs) { - if (input.length() == 0) { - continue; - } - context.doc().add(getCompletionField(ctx, input, suggestPayload)); - } + throw new ElasticsearchParseException("failed to parse expected text or object got" + token.name()); } - return null; - } - - private void checkWeight(long weight) { - if (weight < 0 || weight > Integer.MAX_VALUE) { - throw new IllegalArgumentException("Weight must be in the interval [0..2147483647], but was [" + weight + "]"); - } - } - - public Field getCompletionField(ContextMapping.Context ctx, String input, BytesRef payload) { - final String originalInput = input; - if (input.length() > maxInputLength) { - final int len = correctSubStringLen(input, Math.min(maxInputLength, input.length())); - input = input.substring(0, len); - } - for (int i = 0; i < input.length(); i++) { - if (isReservedChar(input.charAt(i))) { - throw new IllegalArgumentException("Illegal input [" + originalInput + "] UTF-16 codepoint [0x" - + Integer.toHexString((int) input.charAt(i)).toUpperCase(Locale.ROOT) - + "] at position " + i + " is a reserved character"); - } - } - return new SuggestField(fieldType().names().indexName(), ctx, input, fieldType(), payload, fieldType().analyzingSuggestLookupProvider); + completionInputs.add(inputs, weight, contextsMap); } - public static int correctSubStringLen(String input, int len) { - if (Character.isHighSurrogate(input.charAt(len - 1))) { - assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); - return len + 1; - } - return len; - } - - public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { - return fieldType().analyzingSuggestLookupProvider.buildPayload(surfaceForm, weight, payload); - } - - private static final class SuggestField extends Field { - private final BytesRef payload; - private final CompletionTokenStream.ToFiniteStrings toFiniteStrings; - private final ContextMapping.Context ctx; - - public SuggestField(String name, ContextMapping.Context ctx, String value, MappedFieldType type, BytesRef payload, CompletionTokenStream.ToFiniteStrings toFiniteStrings) { - super(name, value, type); - this.payload = payload; - this.toFiniteStrings = toFiniteStrings; - this.ctx = ctx; - } - - @Override - public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) throws IOException { - TokenStream ts = ctx.wrapTokenStream(super.tokenStream(analyzer, previous)); - return new CompletionTokenStream(ts, payload, toFiniteStrings); - } - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(simpleName()) - .field(Fields.TYPE, CONTENT_TYPE); - - builder.field(Fields.ANALYZER, fieldType().indexAnalyzer().name()); + .field(Fields.TYPE.getPreferredName(), CONTENT_TYPE); + builder.field(Fields.ANALYZER.getPreferredName(), fieldType().indexAnalyzer().name()); if (fieldType().indexAnalyzer().name().equals(fieldType().searchAnalyzer().name()) == false) { builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().searchAnalyzer().name()); } - builder.field(Fields.PAYLOADS, fieldType().analyzingSuggestLookupProvider.hasPayloads()); - builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().analyzingSuggestLookupProvider.getPreserveSep()); - builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().analyzingSuggestLookupProvider.getPreservePositionsIncrements()); + CompletionAnalyzer analyzer = (CompletionAnalyzer) fieldType().indexAnalyzer().analyzer(); + builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), analyzer.preserveSep()); + builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), analyzer.preservePositionIncrements()); builder.field(Fields.MAX_INPUT_LENGTH.getPreferredName(), this.maxInputLength); - multiFields.toXContent(builder, params); - if(fieldType().requiresContext()) { - builder.startObject(Fields.CONTEXT); - for (ContextMapping mapping : fieldType().getContextMapping().values()) { - builder.value(mapping); - } - builder.endObject(); + if (fieldType().hasContextMappings()) { + builder.startArray(Fields.CONTEXTS.getPreferredName()); + fieldType().getContextMappings().toXContent(builder, params); + builder.endArray(); } + multiFields.toXContent(builder, params); return builder.endObject(); } @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { + // no-op } @Override @@ -540,10 +492,6 @@ protected String contentType() { return CONTENT_TYPE; } - public boolean isStoringPayloads() { - return fieldType().analyzingSuggestLookupProvider.hasPayloads(); - } - @Override public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { super.merge(mergeWith, mergeResult); @@ -553,21 +501,65 @@ public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMapping } } - // this should be package private but our tests don't allow it. - public static boolean isReservedChar(char character) { - /* we use 0x001F as a SEP_LABEL in the suggester but we can use the UTF-16 representation since they - * are equivalent. We also don't need to convert the input character to UTF-8 here to check for - * the 0x00 end label since all multi-byte UTF-8 chars start with 0x10 binary so if the UTF-16 CP is == 0x00 - * it's the single byte UTF-8 CP */ - assert XAnalyzingSuggester.PAYLOAD_SEP == XAnalyzingSuggester.SEP_LABEL; // ensure they are the same! - switch(character) { - case XAnalyzingSuggester.END_BYTE: - case XAnalyzingSuggester.SEP_LABEL: - case XAnalyzingSuggester.HOLE_CHARACTER: - case ContextMapping.SEPARATOR: - return true; - default: - return false; + private static void addContexts(Map> contextsMap, Map> partialContextsMap) { + for (Map.Entry> context : partialContextsMap.entrySet()) { + Set contexts = contextsMap.get(context.getKey()); + if (contexts == null) { + contexts = context.getValue(); + } else { + contexts.addAll(context.getValue()); + } + contextsMap.put(context.getKey(), contexts); } } + + private static class CompletionInputs implements Iterable> { + Map inputs = Maps.newHashMapWithExpectedSize(4); + + static class CompletionInputMetaData { + public final Map> contexts; + public final int weight; + + CompletionInputMetaData(Map> contexts, int weight) { + this.contexts = contexts; + this.weight = weight; + } + } + + void add(Set inputs, int weight, Map> contexts) { + for (String input : inputs) { + add(input, weight, contexts); + } + } + + void add(String input, int weight, Map> contexts) { + if (inputs.containsKey(input)) { + if (inputs.get(input).weight < weight) { + inputs.put(input, new CompletionInputMetaData(contexts, weight)); + } + } else { + inputs.put(input, new CompletionInputMetaData(contexts, weight)); + } + } + + @Override + public Iterator> iterator() { + return inputs.entrySet().iterator(); + } + } + + public static int correctSubStringLen(String input, int len) { + if (Character.isHighSurrogate(input.charAt(len - 1))) { + assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); + return len + 1; + } + return len; + } + + private static void checkWeight(Number weight) { + if (weight.longValue() < 0 || weight.longValue() > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Weight must be in the interval [0..2147483647], but was [" + weight + "]"); + } + } + } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/OldCompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/OldCompletionFieldMapper.java new file mode 100644 index 0000000000000..d97908e202c08 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/OldCompletionFieldMapper.java @@ -0,0 +1,573 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper.core; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.document.Field; +import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.NumberType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperException; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MergeMappingException; +import org.elasticsearch.index.mapper.MergeResult; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.search.suggest.completion.old.AnalyzingCompletionLookupProvider; +import org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat; +import org.elasticsearch.search.suggest.completion.old.CompletionTokenStream; +import org.elasticsearch.search.suggest.completion.old.context.ContextBuilder; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping.ContextConfig; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; + +import static org.elasticsearch.index.mapper.MapperBuilders.oldCompletionField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; + +/** + * + */ +public class OldCompletionFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "completion"; + + public static class Defaults { + public static final CompletionFieldType FIELD_TYPE = new CompletionFieldType(); + + static { + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.freeze(); + } + + public static final boolean DEFAULT_PRESERVE_SEPARATORS = true; + public static final boolean DEFAULT_POSITION_INCREMENTS = true; + public static final boolean DEFAULT_HAS_PAYLOADS = false; + public static final int DEFAULT_MAX_INPUT_LENGTH = 50; + } + + public static class Fields { + // Mapping field names + public static final String ANALYZER = "analyzer"; + public static final ParseField SEARCH_ANALYZER = new ParseField("search_analyzer"); + public static final ParseField PRESERVE_SEPARATORS = new ParseField("preserve_separators"); + public static final ParseField PRESERVE_POSITION_INCREMENTS = new ParseField("preserve_position_increments"); + public static final String PAYLOADS = "payloads"; + public static final String TYPE = "type"; + public static final ParseField MAX_INPUT_LENGTH = new ParseField("max_input_length", "max_input_len"); + // Content field names + public static final String CONTENT_FIELD_NAME_INPUT = "input"; + public static final String CONTENT_FIELD_NAME_OUTPUT = "output"; + public static final String CONTENT_FIELD_NAME_PAYLOAD = "payload"; + public static final String CONTENT_FIELD_NAME_WEIGHT = "weight"; + public static final String CONTEXT = "context"; + } + + public static final Set ALLOWED_CONTENT_FIELD_NAMES = Sets.newHashSet(Fields.CONTENT_FIELD_NAME_INPUT, + Fields.CONTENT_FIELD_NAME_OUTPUT, Fields.CONTENT_FIELD_NAME_PAYLOAD, Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTEXT); + + public static class Builder extends FieldMapper.Builder { + + private boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS; + private boolean payloads = Defaults.DEFAULT_HAS_PAYLOADS; + private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; + private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH; + private SortedMap contextMapping = ContextMapping.EMPTY_MAPPING; + + public Builder(String name) { + super(name, Defaults.FIELD_TYPE); + builder = this; + } + + public Builder payloads(boolean payloads) { + this.payloads = payloads; + return this; + } + + public Builder preserveSeparators(boolean preserveSeparators) { + this.preserveSeparators = preserveSeparators; + return this; + } + + public Builder preservePositionIncrements(boolean preservePositionIncrements) { + this.preservePositionIncrements = preservePositionIncrements; + return this; + } + + public Builder maxInputLength(int maxInputLength) { + if (maxInputLength <= 0) { + throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() + " must be > 0 but was [" + maxInputLength + "]"); + } + this.maxInputLength = maxInputLength; + return this; + } + + public Builder contextMapping(SortedMap contextMapping) { + this.contextMapping = contextMapping; + return this; + } + + @Override + public OldCompletionFieldMapper build(Mapper.BuilderContext context) { + setupFieldType(context); + CompletionFieldType completionFieldType = (CompletionFieldType)fieldType; + completionFieldType.setProvider(new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, payloads)); + completionFieldType.setContextMapping(contextMapping); + return new OldCompletionFieldMapper(name, fieldType, maxInputLength, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + } + + } + + public static class TypeParser implements Mapper.TypeParser { + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + OldCompletionFieldMapper.Builder builder = oldCompletionField(name); + NamedAnalyzer indexAnalyzer = null; + NamedAnalyzer searchAnalyzer = null; + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (fieldName.equals("type")) { + continue; + } + if (Fields.ANALYZER.equals(fieldName) || // index_analyzer is for backcompat, remove for v3.0 + fieldName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + + indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); + iterator.remove(); + } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.SEARCH_ANALYZER)) { + searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); + iterator.remove(); + } else if (fieldName.equals(Fields.PAYLOADS)) { + builder.payloads(Boolean.parseBoolean(fieldNode.toString())); + iterator.remove(); + } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_SEPARATORS)) { + builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString())); + iterator.remove(); + } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_POSITION_INCREMENTS)) { + builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString())); + iterator.remove(); + } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.MAX_INPUT_LENGTH)) { + builder.maxInputLength(Integer.parseInt(fieldNode.toString())); + iterator.remove(); + } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) { + iterator.remove(); + } else if (fieldName.equals(Fields.CONTEXT)) { + builder.contextMapping(ContextBuilder.loadMappings(fieldNode, parserContext.indexVersionCreated())); + iterator.remove(); + } + } + + if (indexAnalyzer == null) { + if (searchAnalyzer != null) { + throw new MapperParsingException("analyzer on completion field [" + name + "] must be set when search_analyzer is set"); + } + indexAnalyzer = searchAnalyzer = parserContext.analysisService().analyzer("simple"); + } else if (searchAnalyzer == null) { + searchAnalyzer = indexAnalyzer; + } + builder.indexAnalyzer(indexAnalyzer); + builder.searchAnalyzer(searchAnalyzer); + + return builder; + } + + private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) { + NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name); + if (analyzer == null) { + throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]"); + } + return analyzer; + } + } + + public static final class CompletionFieldType extends MappedFieldType { + private PostingsFormat postingsFormat; + private AnalyzingCompletionLookupProvider analyzingSuggestLookupProvider; + private SortedMap contextMapping = ContextMapping.EMPTY_MAPPING; + + public CompletionFieldType() { + setFieldDataType(null); + } + + protected CompletionFieldType(CompletionFieldType ref) { + super(ref); + this.postingsFormat = ref.postingsFormat; + this.analyzingSuggestLookupProvider = ref.analyzingSuggestLookupProvider; + this.contextMapping = ref.contextMapping; + } + + @Override + public CompletionFieldType clone() { + return new CompletionFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public void checkCompatibility(MappedFieldType fieldType, List conflicts, boolean strict) { + super.checkCompatibility(fieldType, conflicts, strict); + CompletionFieldType other = (CompletionFieldType)fieldType; + if (analyzingSuggestLookupProvider.hasPayloads() != other.analyzingSuggestLookupProvider.hasPayloads()) { + conflicts.add("mapper [" + names().fullName() + "] has different payload values"); + } + if (analyzingSuggestLookupProvider.getPreservePositionsIncrements() != other.analyzingSuggestLookupProvider.getPreservePositionsIncrements()) { + conflicts.add("mapper [" + names().fullName() + "] has different 'preserve_position_increments' values"); + } + if (analyzingSuggestLookupProvider.getPreserveSep() != other.analyzingSuggestLookupProvider.getPreserveSep()) { + conflicts.add("mapper [" + names().fullName() + "] has different 'preserve_separators' values"); + } + if(!ContextMapping.mappingsAreEqual(getContextMapping(), other.getContextMapping())) { + conflicts.add("mapper [" + names().fullName() + "] has different 'context_mapping' values"); + } + } + + public void setProvider(AnalyzingCompletionLookupProvider provider) { + checkIfFrozen(); + this.analyzingSuggestLookupProvider = provider; + } + + public synchronized PostingsFormat postingsFormat(PostingsFormat in) { + if (in instanceof Completion090PostingsFormat) { + throw new IllegalStateException("Double wrapping of " + Completion090PostingsFormat.class); + } + if (postingsFormat == null) { + postingsFormat = new Completion090PostingsFormat(in, analyzingSuggestLookupProvider); + } + return postingsFormat; + } + + public void setContextMapping(SortedMap contextMapping) { + checkIfFrozen(); + this.contextMapping = contextMapping; + } + + /** Get the context mapping associated with this completion field */ + public SortedMap getContextMapping() { + return contextMapping; + } + + /** @return true if a context mapping has been defined */ + public boolean requiresContext() { + return contextMapping.isEmpty() == false; + } + + @Override + public String value(Object value) { + if (value == null) { + return null; + } + return value.toString(); + } + + @Override + public boolean isSortable() { + return false; + } + } + + private static final BytesRef EMPTY = new BytesRef(); + + private int maxInputLength; + + public OldCompletionFieldMapper(String simpleName, MappedFieldType fieldType, int maxInputLength, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo); + this.maxInputLength = maxInputLength; + } + + @Override + public CompletionFieldType fieldType() { + return (CompletionFieldType) super.fieldType(); + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + XContentParser parser = context.parser(); + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.VALUE_NULL) { + throw new MapperParsingException("completion field [" + fieldType().names().fullName() + "] does not support null values"); + } + + String surfaceForm = null; + BytesRef payload = null; + long weight = -1; + List inputs = Lists.newArrayListWithExpectedSize(4); + + SortedMap contextConfig = null; + + if (token == XContentParser.Token.VALUE_STRING) { + inputs.add(parser.text()); + multiFields.parse(this, context); + } else { + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (!ALLOWED_CONTENT_FIELD_NAMES.contains(currentFieldName)) { + throw new IllegalArgumentException("Unknown field name[" + currentFieldName + "], must be one of " + ALLOWED_CONTENT_FIELD_NAMES); + } + } else if (Fields.CONTEXT.equals(currentFieldName)) { + SortedMap configs = Maps.newTreeMap(); + + if (token == Token.START_OBJECT) { + while ((token = parser.nextToken()) != Token.END_OBJECT) { + String name = parser.text(); + ContextMapping mapping = fieldType().getContextMapping().get(name); + if (mapping == null) { + throw new ElasticsearchParseException("context [{}] is not defined", name); + } else { + token = parser.nextToken(); + configs.put(name, mapping.parseContext(context, parser)); + } + } + contextConfig = Maps.newTreeMap(); + for (ContextMapping mapping : fieldType().getContextMapping().values()) { + ContextConfig config = configs.get(mapping.name()); + contextConfig.put(mapping.name(), config==null ? mapping.defaultConfig() : config); + } + } else { + throw new ElasticsearchParseException("context must be an object"); + } + } else if (Fields.CONTENT_FIELD_NAME_PAYLOAD.equals(currentFieldName)) { + if (!isStoringPayloads()) { + throw new MapperException("Payloads disabled in mapping"); + } + if (token == XContentParser.Token.START_OBJECT) { + XContentBuilder payloadBuilder = XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser); + payload = payloadBuilder.bytes().toBytesRef(); + payloadBuilder.close(); + } else if (token.isValue()) { + payload = parser.utf8BytesOrNull(); + } else { + throw new MapperException("payload doesn't support type " + token); + } + } else if (token == XContentParser.Token.VALUE_STRING) { + if (Fields.CONTENT_FIELD_NAME_OUTPUT.equals(currentFieldName)) { + surfaceForm = parser.text(); + } + if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { + inputs.add(parser.text()); + } + if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { + Number weightValue; + try { + weightValue = Long.parseLong(parser.text()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Weight must be a string representing a numeric value, but was [" + parser.text() + "]"); + } + weight = weightValue.longValue(); // always parse a long to make sure we don't get overflow + checkWeight(weight); + } + } else if (token == XContentParser.Token.VALUE_NUMBER) { + if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) { + NumberType numberType = parser.numberType(); + if (NumberType.LONG != numberType && NumberType.INT != numberType) { + throw new IllegalArgumentException("Weight must be an integer, but was [" + parser.numberValue() + "]"); + } + weight = parser.longValue(); // always parse a long to make sure we don't get overflow + checkWeight(weight); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + inputs.add(parser.text()); + } + } + } + } + } + + if(contextConfig == null) { + contextConfig = Maps.newTreeMap(); + for (ContextMapping mapping : fieldType().getContextMapping().values()) { + contextConfig.put(mapping.name(), mapping.defaultConfig()); + } + } + + final ContextMapping.Context ctx = new ContextMapping.Context(contextConfig, context.doc()); + + payload = payload == null ? EMPTY : payload; + if (surfaceForm == null) { // no surface form use the input + for (String input : inputs) { + if (input.length() == 0) { + continue; + } + BytesRef suggestPayload = fieldType().analyzingSuggestLookupProvider.buildPayload(new BytesRef( + input), weight, payload); + context.doc().add(getCompletionField(ctx, input, suggestPayload)); + } + } else { + BytesRef suggestPayload = fieldType().analyzingSuggestLookupProvider.buildPayload(new BytesRef( + surfaceForm), weight, payload); + for (String input : inputs) { + if (input.length() == 0) { + continue; + } + context.doc().add(getCompletionField(ctx, input, suggestPayload)); + } + } + return null; + } + + private void checkWeight(long weight) { + if (weight < 0 || weight > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Weight must be in the interval [0..2147483647], but was [" + weight + "]"); + } + } + + public Field getCompletionField(ContextMapping.Context ctx, String input, BytesRef payload) { + final String originalInput = input; + if (input.length() > maxInputLength) { + final int len = correctSubStringLen(input, Math.min(maxInputLength, input.length())); + input = input.substring(0, len); + } + for (int i = 0; i < input.length(); i++) { + if (isReservedChar(input.charAt(i))) { + throw new IllegalArgumentException("Illegal input [" + originalInput + "] UTF-16 codepoint [0x" + + Integer.toHexString((int) input.charAt(i)).toUpperCase(Locale.ROOT) + + "] at position " + i + " is a reserved character"); + } + } + return new SuggestField(fieldType().names().indexName(), ctx, input, fieldType(), payload, fieldType().analyzingSuggestLookupProvider); + } + + public static int correctSubStringLen(String input, int len) { + if (Character.isHighSurrogate(input.charAt(len - 1))) { + assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); + return len + 1; + } + return len; + } + + public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { + return fieldType().analyzingSuggestLookupProvider.buildPayload(surfaceForm, weight, payload); + } + + private static final class SuggestField extends Field { + private final BytesRef payload; + private final CompletionTokenStream.ToFiniteStrings toFiniteStrings; + private final ContextMapping.Context ctx; + + public SuggestField(String name, ContextMapping.Context ctx, String value, MappedFieldType type, BytesRef payload, CompletionTokenStream.ToFiniteStrings toFiniteStrings) { + super(name, value, type); + this.payload = payload; + this.toFiniteStrings = toFiniteStrings; + this.ctx = ctx; + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) throws IOException { + TokenStream ts = ctx.wrapTokenStream(super.tokenStream(analyzer, previous)); + return new CompletionTokenStream(ts, payload, toFiniteStrings); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(simpleName()) + .field(Fields.TYPE, CONTENT_TYPE); + + builder.field(Fields.ANALYZER, fieldType().indexAnalyzer().name()); + if (fieldType().indexAnalyzer().name().equals(fieldType().searchAnalyzer().name()) == false) { + builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().searchAnalyzer().name()); + } + builder.field(Fields.PAYLOADS, fieldType().analyzingSuggestLookupProvider.hasPayloads()); + builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().analyzingSuggestLookupProvider.getPreserveSep()); + builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().analyzingSuggestLookupProvider.getPreservePositionsIncrements()); + builder.field(Fields.MAX_INPUT_LENGTH.getPreferredName(), this.maxInputLength); + multiFields.toXContent(builder, params); + + if(fieldType().requiresContext()) { + builder.startObject(Fields.CONTEXT); + for (ContextMapping mapping : fieldType().getContextMapping().values()) { + builder.value(mapping); + } + builder.endObject(); + } + + return builder.endObject(); + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + public boolean isStoringPayloads() { + return fieldType().analyzingSuggestLookupProvider.hasPayloads(); + } + + @Override + public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException { + super.merge(mergeWith, mergeResult); + OldCompletionFieldMapper fieldMergeWith = (OldCompletionFieldMapper) mergeWith; + if (!mergeResult.simulate()) { + this.maxInputLength = fieldMergeWith.maxInputLength; + } + } + + // this should be package private but our tests don't allow it. + public static boolean isReservedChar(char character) { + /* we use 0x001F as a SEP_LABEL in the suggester but we can use the UTF-16 representation since they + * are equivalent. We also don't need to convert the input character to UTF-8 here to check for + * the 0x00 end label since all multi-byte UTF-8 chars start with 0x10 binary so if the UTF-16 CP is == 0x00 + * it's the single byte UTF-8 CP */ + assert XAnalyzingSuggester.PAYLOAD_SEP == XAnalyzingSuggester.SEP_LABEL; // ensure they are the same! + switch(character) { + case XAnalyzingSuggester.END_BYTE: + case XAnalyzingSuggester.SEP_LABEL: + case XAnalyzingSuggester.HOLE_CHARACTER: + case ContextMapping.SEPARATOR: + return true; + default: + return false; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java index 0af6b86db304d..f105ad9bb92d7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java @@ -108,7 +108,7 @@ public int value() { * @param flags A string representing a list of regualr expression flags * @return The combined OR'ed value for all the flags */ - static int resolveValue(String flags) { + public static int resolveValue(String flags) { if (flags == null || flags.isEmpty()) { return RegExp.ALL; } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index bb681586781a2..d2ffcd9357a38 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -104,8 +104,9 @@ import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat; +import org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat; import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.search.suggest.completion.CompletionFieldStats; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -658,15 +659,10 @@ public SuggestStats suggestStats() { public CompletionStats completionStats(String... fields) { CompletionStats completionStats = new CompletionStats(); - final Engine.Searcher currentSearcher = acquireSearcher("completion_stats"); - try { - PostingsFormat postingsFormat = PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME); - if (postingsFormat instanceof Completion090PostingsFormat) { - Completion090PostingsFormat completionPostingsFormat = (Completion090PostingsFormat) postingsFormat; - completionStats.add(completionPostingsFormat.completionStats(currentSearcher.reader(), fields)); - } - } finally { - currentSearcher.close(); + try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) { + Completion090PostingsFormat postingsFormat = ((Completion090PostingsFormat) PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME)); + completionStats.add(postingsFormat.completionStats(currentSearcher.reader(), fields)); + completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields)); } return completionStats; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java index 8a1f5f1263639..a4920e6826ffc 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggest.java @@ -121,6 +121,9 @@ public void readFrom(StreamInput in) throws IOException { case CompletionSuggestion.TYPE: suggestion = new CompletionSuggestion(); break; + case org.elasticsearch.search.suggest.completion.old.CompletionSuggestion.TYPE: + suggestion = new org.elasticsearch.search.suggest.completion.old.CompletionSuggestion(); + break; case PhraseSuggestion.TYPE: suggestion = new PhraseSuggestion(); break; @@ -181,6 +184,15 @@ public static List>> reduce(Map>> reduced = new ArrayList<>(groupedSuggestions.size()); for (java.util.Map.Entry> unmergedResults : groupedSuggestions.entrySet()) { List value = unmergedResults.getValue(); + Class suggestionClass = null; + for (Suggestion suggestion : value) { + if (suggestionClass == null) { + suggestionClass = suggestion.getClass(); + } else if (suggestionClass != suggestion.getClass()) { + throw new IllegalArgumentException("detected mixed suggestion results, due to querying on old and new completion suggester," + + " query on a single completion suggester version"); + } + } Suggestion reduce = value.get(0).reduce(value); reduce.trim(); reduced.add(reduce); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java index 57d18c3cddbea..82d0cc570a5b6 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java @@ -20,9 +20,9 @@ import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.search.suggest.context.CategoryContextMapping; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; -import org.elasticsearch.search.suggest.context.GeolocationContextMapping; +import org.elasticsearch.search.suggest.completion.old.context.CategoryContextMapping; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping.ContextQuery; +import org.elasticsearch.search.suggest.completion.old.context.GeolocationContextMapping; import java.io.IOException; import java.util.ArrayList; @@ -101,11 +101,14 @@ public static abstract class SuggestionBuilder extends ToXContentToBytes { private String name; private String suggester; private String text; + private String prefix; + private String regex; private String field; private String analyzer; private Integer size; private Integer shardSize; - + + // TODO: remove private List contextQueries = new ArrayList<>(); public SuggestionBuilder(String name, String suggester) { @@ -113,6 +116,7 @@ public SuggestionBuilder(String name, String suggester) { this.suggester = suggester; } + // TODO: remove, these should be in CompletionSuggestionBuilder @SuppressWarnings("unchecked") private T addContextQuery(ContextQuery ctx) { this.contextQueries.add(ctx); @@ -125,6 +129,7 @@ private T addContextQuery(ContextQuery ctx) { * @param lon Longitude of the Location * @return this */ + @Deprecated public T addGeoLocation(String name, double lat, double lon, int ... precisions) { return addContextQuery(GeolocationContextMapping.query(name, lat, lon, precisions)); } @@ -136,6 +141,7 @@ public T addGeoLocation(String name, double lat, double lon, int ... precisions) * @param precisions precisions as string var-args * @return this */ + @Deprecated public T addGeoLocationWithPrecision(String name, double lat, double lon, String ... precisions) { return addContextQuery(GeolocationContextMapping.query(name, lat, lon, precisions)); } @@ -145,6 +151,7 @@ public T addGeoLocationWithPrecision(String name, double lat, double lon, String * @param geohash Geohash of the location * @return this */ + @Deprecated public T addGeoLocation(String name, String geohash) { return addContextQuery(GeolocationContextMapping.query(name, geohash)); } @@ -154,6 +161,7 @@ public T addGeoLocation(String name, String geohash) { * @param categories name of the category * @return this */ + @Deprecated public T addCategory(String name, CharSequence...categories) { return addContextQuery(CategoryContextMapping.query(name, categories)); } @@ -163,6 +171,7 @@ public T addCategory(String name, CharSequence...categories) { * @param categories name of the category * @return this */ + @Deprecated public T addCategory(String name, Iterable categories) { return addContextQuery(CategoryContextMapping.query(name, categories)); } @@ -172,6 +181,7 @@ public T addCategory(String name, Iterable categories) { * @param fieldvalues name of the category * @return this */ + @Deprecated public T addContextField(String name, CharSequence...fieldvalues) { return addContextQuery(CategoryContextMapping.query(name, fieldvalues)); } @@ -181,6 +191,7 @@ public T addContextField(String name, CharSequence...fieldvalues) { * @param fieldvalues name of the category * @return this */ + @Deprecated public T addContextField(String name, Iterable fieldvalues) { return addContextQuery(CategoryContextMapping.query(name, fieldvalues)); } @@ -194,12 +205,26 @@ public T text(String text) { return (T) this; } + protected void setPrefix(String prefix) { + this.prefix = prefix; + } + + protected void setRegex(String regex) { + this.regex = regex; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(name); if (text != null) { builder.field("text", text); } + if (prefix != null) { + builder.field("prefix", prefix); + } + if (regex != null) { + builder.field("regex", regex); + } builder.startObject(suggester); if (analyzer != null) { builder.field("analyzer", analyzer); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java index 16957986e2789..82cddf5d14d30 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilders.java @@ -19,8 +19,8 @@ package org.elasticsearch.search.suggest; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestionFuzzyBuilder; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; @@ -55,21 +55,34 @@ public static PhraseSuggestionBuilder phraseSuggestion(String name) { * Creates a completion suggestion lookup query with the provided name * * @param name The suggestion name - * @return a {@link org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder} + * @return a {@link org.elasticsearch.search.suggest.completion.old.CompletionSuggestionBuilder} * instance */ public static CompletionSuggestionBuilder completionSuggestion(String name) { return new CompletionSuggestionBuilder(name); } + /** + * Creates a completion suggestion lookup query with the provided name + * + * @param name The suggestion name + * @return a {@link org.elasticsearch.search.suggest.completion.old.CompletionSuggestionBuilder} + * instance + */ + @Deprecated + public static org.elasticsearch.search.suggest.completion.old.CompletionSuggestionBuilder oldCompletionSuggestion(String name) { + return new org.elasticsearch.search.suggest.completion.old.CompletionSuggestionBuilder(name); + } + /** * Creates a fuzzy completion suggestion lookup query with the provided name * * @param name The suggestion name - * @return a {@link org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder} + * @return a {@link CompletionSuggestionFuzzyBuilder} * instance */ - public static CompletionSuggestionFuzzyBuilder fuzzyCompletionSuggestion(String name) { + @Deprecated + public static CompletionSuggestionFuzzyBuilder oldFuzzyCompletionSuggestion(String name) { return new CompletionSuggestionFuzzyBuilder(name); } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestModule.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestModule.java index c01ab3a7afa0c..b49eee72257d0 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestModule.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestModule.java @@ -37,6 +37,9 @@ public class SuggestModule extends AbstractModule { public SuggestModule() { registerSuggester(PhraseSuggester.class); registerSuggester(TermSuggester.class); + // NOTE: the old completion suggester is registered only to access the old + // suggester in SuggestParseElement as "completion_old" + registerSuggester(org.elasticsearch.search.suggest.completion.old.CompletionSuggester.class); registerSuggester(CompletionSuggester.class); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java index 637ed3d6c4810..74ddf6a049812 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java @@ -26,6 +26,8 @@ import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; +import org.elasticsearch.search.suggest.completion.CompletionSuggestParser; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggester; import java.io.IOException; import java.util.Map; @@ -69,6 +71,8 @@ public SuggestionSearchContext parseInternal(XContentParser parser, MapperServic } else if (token == XContentParser.Token.START_OBJECT) { String suggestionName = fieldName; BytesRef suggestText = null; + BytesRef prefix = null; + BytesRef regex = null; SuggestionContext suggestionContext = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -77,6 +81,10 @@ public SuggestionSearchContext parseInternal(XContentParser parser, MapperServic } else if (token.isValue()) { if ("text".equals(fieldName)) { suggestText = parser.utf8Bytes(); + } else if ("prefix".equals(fieldName)) { + prefix = parser.utf8Bytes(); + } else if ("regex".equals(fieldName)) { + regex = parser.utf8Bytes(); } else { throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]"); } @@ -88,11 +96,23 @@ public SuggestionSearchContext parseInternal(XContentParser parser, MapperServic throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported"); } final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser(); + if (contextParser instanceof CompletionSuggestParser) { + ((CompletionSuggestParser) contextParser).setOldCompletionSuggester(((CompletionSuggester) suggesters.get("completion_old"))); + } suggestionContext = contextParser.parse(parser, mapperService, queryParserService); } } if (suggestionContext != null) { - suggestionContext.setText(suggestText); + if (suggestText != null && prefix == null) { + suggestionContext.setPrefix(suggestText); + suggestionContext.setText(suggestText); + } else if (suggestText == null && prefix != null) { + suggestionContext.setPrefix(prefix); + suggestionContext.setText(prefix); + } else if (regex != null) { + suggestionContext.setRegex(regex); + suggestionContext.setText(regex); + } suggestionContexts.put(suggestionName, suggestionContext); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java index 2cb36f5391453..1d3339e0578ba 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java @@ -40,6 +40,8 @@ public Map suggestions() { public static class SuggestionContext { private BytesRef text; + private BytesRef prefix; + private BytesRef regex; private final Suggester suggester; private String field; private Analyzer analyzer; @@ -55,7 +57,23 @@ public BytesRef getText() { public void setText(BytesRef text) { this.text = text; } - + + public BytesRef getPrefix() { + return prefix; + } + + public void setPrefix(BytesRef prefix) { + this.prefix = prefix; + } + + public BytesRef getRegex() { + return regex; + } + + public void setRegex(BytesRef regex) { + this.regex = regex; + } + public SuggestionContext(Suggester suggester) { this.suggester = suggester; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java new file mode 100644 index 0000000000000..65b4b03dc478d --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionFieldStats.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion; + +import com.carrotsearch.hppc.ObjectLongHashMap; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.suggest.xdocument.CompletionTerms; +import org.elasticsearch.common.regex.Regex; + +import java.io.IOException; + +public class CompletionFieldStats { + + public static CompletionStats completionStats(IndexReader indexReader, String ... fields) { + long sizeInBytes = 0; + ObjectLongHashMap completionFields = null; + if (fields != null && fields.length > 0) { + completionFields = new ObjectLongHashMap<>(fields.length); + } + for (LeafReaderContext atomicReaderContext : indexReader.leaves()) { + LeafReader atomicReader = atomicReaderContext.reader(); + try { + for (String fieldName : atomicReader.fields()) { + Terms terms = atomicReader.fields().terms(fieldName); + if (terms instanceof CompletionTerms) { + long fstSize = ((CompletionTerms) terms).ramBytesUsed(); + if (fields != null && fields.length > 0 && Regex.simpleMatch(fields, fieldName)) { + completionFields.addTo(fieldName, fstSize); + } + sizeInBytes += fstSize; + } + } + } catch (IOException ignored) { + } + } + return new CompletionStats(sizeInBytes, completionFields); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java index 9802db38130da..c861e104279be 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java @@ -18,105 +18,214 @@ */ package org.elasticsearch.search.suggest.completion; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.query.RegexpFlag; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; +import org.elasticsearch.search.suggest.completion.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; +import org.elasticsearch.search.suggest.completion.context.ContextMappingsParser; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Map; import static org.elasticsearch.search.suggest.SuggestUtils.parseSuggestContext; +import static org.elasticsearch.search.suggest.completion.context.ContextMappingsParser.parseQueryContext; /** + * Parses query options for {@link CompletionSuggester} * + * Acceptable input: + * { + * "field" : STRING + * "size" : INT + * "fuzzy" : BOOLEAN | FUZZY_OBJECT + * "contexts" : QUERY_CONTEXTS + * "regex" : REGEX_OBJECT + * } + * + * see {@link ContextMappingsParser#parseQueryContext(ContextMappings, XContentParser)} for QUERY_CONTEXTS + * + * FUZZY_OBJECT : { + * "edit_distance" : STRING | INT + * "transpositions" : BOOLEAN + * "min_length" : INT + * "prefix_length" : INT + * "unicode_aware" : BOOLEAN + * "max_determinized_states" : INT + * } + * + * REGEX_OBJECT: { + * "flags" : REGEX_FLAGS + * "max_determinized_states" : INT + * } + * + * see {@link RegexpFlag} for REGEX_FLAGS */ public class CompletionSuggestParser implements SuggestContextParser { private CompletionSuggester completionSuggester; private static final ParseField FUZZINESS = Fuzziness.FIELD.withDeprecation("edit_distance"); + private org.elasticsearch.search.suggest.completion.old.CompletionSuggester oldCompletionSuggester; public CompletionSuggestParser(CompletionSuggester completionSuggester) { this.completionSuggester = completionSuggester; } @Override - public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexQueryParserService queryParserService) throws IOException { + public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, + IndexQueryParserService queryParserService) throws IOException { XContentParser.Token token; String fieldName = null; CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester); XContentParser contextParser = null; - + CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptions = null; + CompletionSuggestionBuilder.RegexOptionsBuilder regexOptions = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { fieldName = parser.currentName(); } else if (token.isValue()) { - if (!parseSuggestContext(parser, mapperService, fieldName, suggestion, queryParserService.parseFieldMatcher())) { + if (!parseSuggestContext(parser, mapperService, fieldName, suggestion, queryParserService.parseFieldMatcher())) { if (token == XContentParser.Token.VALUE_BOOLEAN && "fuzzy".equals(fieldName)) { - suggestion.setFuzzy(parser.booleanValue()); + if (parser.booleanValue()) { + fuzzyOptions = new CompletionSuggestionBuilder.FuzzyOptionsBuilder(); + } } } } else if (token == XContentParser.Token.START_OBJECT) { - if("fuzzy".equals(fieldName)) { - suggestion.setFuzzy(true); + if ("fuzzy".equals(fieldName)) { + fuzzyOptions = new CompletionSuggestionBuilder.FuzzyOptionsBuilder(); String fuzzyConfigName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { fuzzyConfigName = parser.currentName(); } else if (token.isValue()) { if (queryParserService.parseFieldMatcher().match(fuzzyConfigName, FUZZINESS)) { - suggestion.setFuzzyEditDistance(Fuzziness.parse(parser).asDistance()); + fuzzyOptions.setFuzziness(Fuzziness.parse(parser).asDistance()); } else if ("transpositions".equals(fuzzyConfigName)) { - suggestion.setFuzzyTranspositions(parser.booleanValue()); + fuzzyOptions.setTranspositions(parser.booleanValue()); } else if ("min_length".equals(fuzzyConfigName) || "minLength".equals(fuzzyConfigName)) { - suggestion.setFuzzyMinLength(parser.intValue()); + fuzzyOptions.setFuzzyMinLength(parser.intValue()); } else if ("prefix_length".equals(fuzzyConfigName) || "prefixLength".equals(fuzzyConfigName)) { - suggestion.setFuzzyPrefixLength(parser.intValue()); + fuzzyOptions.setFuzzyPrefixLength(parser.intValue()); } else if ("unicode_aware".equals(fuzzyConfigName) || "unicodeAware".equals(fuzzyConfigName)) { - suggestion.setFuzzyUnicodeAware(parser.booleanValue()); + fuzzyOptions.setUnicodeAware(parser.booleanValue()); + } else if ("max_determinized_states".equals(fuzzyConfigName)) { + fuzzyOptions.setMaxDeterminizedStates(parser.intValue()); + } else { + throw new IllegalArgumentException("[fuzzy] query does not support [" + fuzzyConfigName + "]"); } } } - } else if("context".equals(fieldName)) { + } else if ("contexts".equals(fieldName) || "context".equals(fieldName)) { // Copy the current structure. We will parse, once the mapping is provided XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); builder.copyCurrentStructure(parser); - BytesReference bytes = builder.bytes(); - contextParser = parser.contentType().xContent().createParser(bytes); + BytesReference bytes = builder.bytes(); + contextParser = XContentFactory.xContent(bytes).createParser(bytes); + } else if ("regex".equals(fieldName)) { + regexOptions = new CompletionSuggestionBuilder.RegexOptionsBuilder(); + String currentFieldName = fieldName; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("flags".equals(currentFieldName)) { + String flags = parser.textOrNull(); + regexOptions.setFlags(flags); + } else if ("max_determinized_states".equals(currentFieldName)) { + regexOptions.setMaxDeterminizedStates(parser.intValue()); + } else { + throw new IllegalArgumentException("[regexp] query does not support [" + currentFieldName + "]"); + } + } + } } else { throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]"); } } else { - throw new IllegalArgumentException("suggester[completion] doesn't support field [" + fieldName + "]"); + throw new IllegalArgumentException("suggester [completion] doesn't support field [" + fieldName + "]"); } } - - suggestion.fieldType((CompletionFieldMapper.CompletionFieldType) mapperService.smartNameFieldType(suggestion.getField())); + MappedFieldType mappedFieldType = mapperService.smartNameFieldType(suggestion.getField()); + if (mappedFieldType == null) { + throw new ElasticsearchException("Field [" + suggestion.getField() + "] is not a completion suggest field"); + } else if (mappedFieldType instanceof CompletionFieldMapper.CompletionFieldType) { + CompletionFieldMapper.CompletionFieldType type = (CompletionFieldMapper.CompletionFieldType) mappedFieldType; + if (type.hasContextMappings() == false && contextParser != null) { + throw new IllegalArgumentException("suggester [" + type.names().fullName() + "] doesn't expect any context"); + } + Map queryContexts = Collections.emptyMap(); + if (type.hasContextMappings() && contextParser != null) { + contextParser.nextToken(); + queryContexts = parseQueryContext(type.getContextMappings(), contextParser); + contextParser.close(); + } + + suggestion.fieldType(type); + suggestion.setFuzzyOptionsBuilder(fuzzyOptions); + suggestion.setRegexOptionsBuilder(regexOptions); + suggestion.setQueryContexts(queryContexts); + // TODO: pass a query builder or the query itself? + // now we do it in CompletionSuggester#toQuery(CompletionSuggestionContext) + return suggestion; - CompletionFieldMapper.CompletionFieldType fieldType = suggestion.fieldType(); - if (fieldType != null) { + } else if (mappedFieldType instanceof OldCompletionFieldMapper.CompletionFieldType) { + org.elasticsearch.search.suggest.completion.old.CompletionSuggestionContext oldSuggestionContext = + new org.elasticsearch.search.suggest.completion.old.CompletionSuggestionContext(oldCompletionSuggester); + OldCompletionFieldMapper.CompletionFieldType fieldType = (OldCompletionFieldMapper.CompletionFieldType) mappedFieldType; + oldSuggestionContext.setAnalyzer(suggestion.getAnalyzer()); + oldSuggestionContext.setField(suggestion.getField()); + oldSuggestionContext.setSize(suggestion.getSize()); + if (suggestion.getShardSize() > 0) { + oldSuggestionContext.setShardSize(suggestion.getShardSize()); + } + + if (fuzzyOptions != null) { + oldSuggestionContext.setFuzzy(true); + oldSuggestionContext.setFuzzyEditDistance(fuzzyOptions.getEditDistance()); + oldSuggestionContext.setFuzzyMinLength(fuzzyOptions.getFuzzyMinLength()); + oldSuggestionContext.setFuzzyPrefixLength(fuzzyOptions.getFuzzyPrefixLength()); + oldSuggestionContext.setFuzzyTranspositions(fuzzyOptions.isTranspositions()); + oldSuggestionContext.setFuzzyUnicodeAware(fuzzyOptions.isUnicodeAware()); + } + + oldSuggestionContext.fieldType(fieldType); if (fieldType.requiresContext()) { if (contextParser == null) { throw new IllegalArgumentException("suggester [completion] requires context to be setup"); } else { contextParser.nextToken(); - List contextQueries = ContextQuery.parseQueries(fieldType.getContextMapping(), contextParser); - suggestion.setContextQuery(contextQueries); + List contextQueries = + org.elasticsearch.search.suggest.completion.old.context.ContextMapping.ContextQuery.parseQueries(fieldType.getContextMapping(), contextParser); + contextParser.close(); + oldSuggestionContext.setContextQuery(contextQueries); } } else if (contextParser != null) { throw new IllegalArgumentException("suggester [completion] doesn't expect any context"); } + return oldSuggestionContext; + } else { + throw new ElasticsearchException("Field [" + suggestion.getField() + "] is not a completion suggest field"); } - return suggestion; } + public void setOldCompletionSuggester(org.elasticsearch.search.suggest.completion.old.CompletionSuggester oldCompletionSuggester) { + this.oldCompletionSuggester = oldCompletionSuggester; + } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index 4af360fa05ff2..dc10493eaf51e 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -18,34 +18,39 @@ */ package org.elasticsearch.search.suggest.completion; -import com.google.common.collect.Maps; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Terms; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.suggest.xdocument.CompletionQuery; +import org.apache.lucene.search.suggest.xdocument.TopSuggestDocs; +import org.apache.lucene.search.suggest.xdocument.TopSuggestDocsCollector; import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.text.StringText; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestContextParser; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option; +import org.elasticsearch.search.suggest.completion.context.ContextMappings; import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; +import java.util.*; public class CompletionSuggester extends Suggester { - private static final ScoreComparator scoreComparator = new ScoreComparator(); + @Override + public String[] names() { + return new String[] { "completion" }; + } + @Override + public SuggestContextParser getContextParser() { + return new CompletionSuggestParser(this); + } @Override protected Suggest.Suggestion> innerExecute(String name, @@ -53,68 +58,99 @@ protected Suggest.Suggestion results = Maps.newHashMapWithExpectedSize(indexReader.leaves().size() * suggestionContext.getSize()); - for (LeafReaderContext atomicReaderContext : indexReader.leaves()) { - LeafReader atomicReader = atomicReaderContext.reader(); - Terms terms = atomicReader.fields().terms(fieldName); - if (terms instanceof Completion090PostingsFormat.CompletionTerms) { - final Completion090PostingsFormat.CompletionTerms lookupTerms = (Completion090PostingsFormat.CompletionTerms) terms; - final Lookup lookup = lookupTerms.getLookup(suggestionContext.fieldType(), suggestionContext); - if (lookup == null) { - // we don't have a lookup for this segment.. this might be possible if a merge dropped all - // docs from the segment that had a value in this segment. - continue; - } - List lookupResults = lookup.lookup(spare.get(), false, suggestionContext.getSize()); - for (Lookup.LookupResult res : lookupResults) { - - final String key = res.key.toString(); - final float score = res.value; - final Option value = results.get(key); - if (value == null) { - final Option option = new CompletionSuggestion.Entry.Option(new StringText(key), score, res.payload == null ? null - : new BytesArray(res.payload)); - results.put(key, option); - } else if (value.getScore() < score) { - value.setScore(score); - value.setPayload(res.payload == null ? null : new BytesArray(res.payload)); - } + Map results = new LinkedHashMap<>(suggestionContext.getSize()); + TopSuggestDocsCollector collector = new TopSuggestDocsCollector(suggestionContext.getSize()); + suggest(searcher, toQuery(suggestionContext), collector); + for (TopSuggestDocs.SuggestScoreDoc suggestDoc : collector.get().scoreLookupDocs()) { + // TODO: currently we can get multiple entries with the same docID + // this has to be fixed at the lucene level + // This has other implications: + // if we index a suggestion with n contexts, the suggestion and all its contexts + // would count as n hits rather than 1, so we have to multiply the desired size + // with n to get a suggestion with all n contexts + final String key = suggestDoc.key.toString(); + final float score = suggestDoc.score; + final Map.Entry contextEntry; + if (suggestionContext.fieldType().hasContextMappings() && suggestDoc.context != null) { + contextEntry = suggestionContext.fieldType().getContextMappings().getNamedContext(suggestDoc.context); + } else { + assert suggestDoc.context == null; + contextEntry = null; + } + final Option value = results.get(suggestDoc.doc); + if (value == null) { + final Option option = new Option(suggestDoc.doc, new StringText(key), score, contextEntry); + results.put(suggestDoc.doc, option); + } else { + value.addContextEntry(contextEntry); + if (value.getScore() < score) { + value.setScore(score); } } } - final List options = new ArrayList<>(results.values()); - CollectionUtil.introSort(options, scoreComparator); - + final List

      + *
    • Array: 
      [<string>, ..]
      • + *
    • String: 
      "string"
      • + *
    + */ + @Override + public Set parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException { + final Set contexts = new HashSet<>(); + Token token = parser.currentToken(); + if (token == Token.VALUE_STRING) { + contexts.add(parser.text()); + } else if (token == Token.START_ARRAY) { + while(parser.nextToken() != Token.END_ARRAY) { + contexts.add(parser.text()); + } + } else { + throw new ElasticsearchParseException("Category contexts must be a string or a list of strings"); + } + return contexts; + } + + @Override + public Set parseContext(Document document) { + Set values = null; + if (fieldName != null) { + IndexableField[] fields = document.getFields(fieldName); + values = new HashSet<>(fields.length); + for (IndexableField field : fields) { + values.add(field.stringValue()); + } + } + return (values == null) ? new HashSet(0) : values; + } + + /** + * Parse a {@link QueryContexts} + * using parser. A QueryContexts accepts one of the following forms: + * + *
      + *
    • Object: CategoryQueryContext
      • + *
    • String: CategoryQueryContext value with prefix=false and boost=1
      • + *
    • Array: 
      [CategoryQueryContext, ..]
      • + *
    + * + * A CategoryQueryContext has one of the following forms: + *
      + *
    • Object: 
      {"context": <string>, "boost": <int>, "prefix": <boolean>}
      • + *
    • String: 
      "string"
      • + *
    + */ + @Override + public QueryContexts parseQueryContext(String name, XContentParser parser) throws IOException, ElasticsearchParseException { + final QueryContexts queryContexts = new QueryContexts<>(name); + Token token = parser.nextToken(); + if (token == Token.START_OBJECT || token == Token.VALUE_STRING) { + queryContexts.add(innerParseQueryContext(parser)); + } else if (token == Token.START_ARRAY) { + while (parser.nextToken() != Token.END_ARRAY) { + queryContexts.add(innerParseQueryContext(parser)); + } + } + return queryContexts; + } + + private CategoryQueryContext innerParseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException { + Token token = parser.currentToken(); + if (token == Token.VALUE_STRING) { + return new CategoryQueryContext(parser.text()); + } else if (token == Token.START_OBJECT) { + String currentFieldName = null; + String context = null; + boolean isPrefix = false; + int boost = 1; + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == Token.VALUE_STRING) { + // context, exact + if (CONTEXT_VALUE.equals(currentFieldName)) { + context = parser.text(); + } else if (CONTEXT_PREFIX.equals(currentFieldName)) { + isPrefix = Boolean.valueOf(parser.text()); + } else if (CONTEXT_BOOST.equals(currentFieldName)) { + Number number; + try { + number = Long.parseLong(parser.text()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Boost must be a string representing a numeric value, but was [" + parser.text() + "]"); + } + boost = number.intValue(); + } + } else if (token == Token.VALUE_NUMBER) { + // boost + if (CONTEXT_BOOST.equals(currentFieldName)) { + Number number = parser.numberValue(); + if (parser.numberType() == XContentParser.NumberType.INT) { + boost = number.intValue(); + } else { + throw new ElasticsearchParseException("Boost must be in the interval [0..2147483647], but was [" + number.longValue() + "]"); + } + } + } else if (token == Token.VALUE_BOOLEAN) { + // exact + if (CONTEXT_PREFIX.equals(currentFieldName)) { + isPrefix = parser.booleanValue(); + } + } + } + if (context == null) { + throw new ElasticsearchParseException("no context provided"); + } + return new CategoryQueryContext(context, boost, isPrefix); + } else { + throw new ElasticsearchParseException("expected string or object"); + } + } + + @Override + protected void addQueryContexts(ContextQuery query, QueryContexts queryContexts) { + for (CategoryQueryContext queryContext : queryContexts) { + query.addContext(queryContext.context, queryContext.boost, queryContext.isPrefix == false); + } + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + if (obj instanceof CategoryContextMapping) { + CategoryContextMapping other = (CategoryContextMapping) obj; + return (this.fieldName.equals(other.fieldName)); + } + } + return false; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((fieldName == null) ? 0 : fieldName.hashCode()); + return result; + } + + /** + * Builder for {@link CategoryContextMapping} + */ + public static class Builder extends ContextBuilder { + + private String fieldName; + + /** + * Create a builder for + * a named {@link CategoryContextMapping} + * @param name name of the mapping + */ + public Builder(String name) { + super(name); + } + + /** + * Set the name of the field to use + */ + public Builder field(String fieldName) { + this.fieldName = fieldName; + return this; + } + + @Override + public CategoryContextMapping build() { + return new CategoryContextMapping(name, fieldName); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java new file mode 100644 index 0000000000000..ae928660c2293 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_BOOST; +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_PREFIX; +import static org.elasticsearch.search.suggest.completion.context.CategoryContextMapping.CONTEXT_VALUE; + +/** + * Defines the query context for {@link CategoryContextMapping} + */ +public class CategoryQueryContext implements ToXContent { + /** + * The value of the category + */ + public final CharSequence context; + + /** + * Whether the value is a + * prefix of a category value or not + */ + public final boolean isPrefix; + + /** + * Query-time boost to be + * applied to suggestions associated + * with this category + */ + public final int boost; + + /** + * Creates a query context with a provided context and a + * boost of 1 + */ + public CategoryQueryContext(CharSequence context) { + this(context, 1); + } + + /** + * Creates a query context with a provided context and boost + */ + public CategoryQueryContext(CharSequence context, int boost) { + this(context, boost, false); + } + + /** + * Creates a query context with a provided context and boost + * Allows specifying whether the context should be treated as + * a prefix or not + */ + public CategoryQueryContext(CharSequence context, int boost, boolean isPrefix) { + this.context = context; + this.boost = boost; + this.isPrefix = isPrefix; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONTEXT_VALUE, context); + builder.field(CONTEXT_BOOST, boost); + builder.field(CONTEXT_PREFIX, isPrefix); + builder.endObject(); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextBuilder.java new file mode 100644 index 0000000000000..9e31d8370cbe3 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextBuilder.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +/** + * Builder for {@link ContextMapping} + */ +public abstract class ContextBuilder { + + protected String name; + + /** + * @param name of the context mapper to build + */ + protected ContextBuilder(String name) { + this.name = name; + } + + public abstract E build(); + + /** + * Create a new {@link GeoContextMapping} + */ + public static GeoContextMapping.Builder geo(String name) { + return new GeoContextMapping.Builder(name); + } + + /** + * Create a new {@link CategoryContextMapping} + */ + public static CategoryContextMapping.Builder category(String name) { + return new CategoryContextMapping.Builder(name); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java new file mode 100644 index 0000000000000..1b96b765e96a9 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMapping.java @@ -0,0 +1,211 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.apache.lucene.search.suggest.xdocument.ContextQuery; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.CompletionFieldMapper; + +import java.io.IOException; +import java.util.*; + +/** + * A {@link ContextMapping} defines criteria that can be used to + * filter and/or boost suggestions at query time for {@link CompletionFieldMapper}. + * + * Implementations have to define how contexts are parsed at query/index time + * (used by {@link ContextMappingsParser}) and add parsed query contexts to query + * supplied by {@link ContextMappings} + * + * @param query context representation + */ +public abstract class ContextMapping implements ToXContent { + + public static final String FIELD_TYPE = "type"; + public static final String FIELD_NAME = "name"; + protected final Type type; + protected final String name; + + public enum Type { + CATEGORY, GEO; + + public static Type fromString(String type) { + if (type.equalsIgnoreCase("category")) { + return CATEGORY; + } else if (type.equalsIgnoreCase("geo")) { + return GEO; + } else { + throw new IllegalArgumentException("No context type for [" + type + "]"); + } + } + } + + /** + * Define a new context mapping of a specific type + * + * @param type type of context mapping, either {@link Type#CATEGORY} or {@link Type#GEO} + * @param name name of context mapping + */ + protected ContextMapping(Type type, String name) { + super(); + this.type = type; + this.name = name; + } + + /** + * @return the type name of the context + */ + public Type type() { + return type; + } + + /** + * @return the name/id of the context + */ + public String name() { + return name; + } + + /** + * Parses a set of index-time contexts. + */ + protected abstract Set parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException; + + /** + * Retrieves a set of context from a document at index-time. + */ + protected abstract Set parseContext(ParseContext.Document document); + + /** + * Parses query contexts for this mapper + */ + protected abstract QueryContexts parseQueryContext(String name, XContentParser parser) throws IOException, ElasticsearchParseException; + + /** + * Named holder for a set of query context + * @param query context type + */ + public static class QueryContexts implements Iterable, ToXContent { + private String name; + private List queryContexts; + + /** + * Constructs a query contexts holder + * for a context mapping + * @param name name of the context mapping to + * query against + */ + public QueryContexts(String name) { + this.name = name; + this.queryContexts = new ArrayList<>(); + } + + /** + * @return name of the context mapping + * to query against + */ + public String getName() { + return name; + } + + /** + * Adds a query context to the holder + * @param queryContext instance + */ + public void add(T queryContext) { + this.queryContexts.add(queryContext); + } + + /** + * @return the number of query contexts + * added + */ + public int size() { + return queryContexts.size(); + } + + @Override + public Iterator iterator() { + return queryContexts.iterator(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray(name); + for (T queryContext : queryContexts) { + queryContext.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + } + + /** + * Adds query contexts to a completion query + */ + protected abstract void addQueryContexts(ContextQuery query, QueryContexts queryContexts); + + /** + * Implementations should add specific configurations + * that need to be persisted + */ + protected abstract XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException; + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FIELD_NAME, name); + builder.field(FIELD_TYPE, type.name()); + toInnerXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + name.hashCode(); + result = prime * result + type.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || (obj instanceof ContextMapping) == false) { + return false; + } + ContextMapping other = ((ContextMapping) obj); + return name.equals(other.name) && type == other.type; + } + + @Override + public String toString() { + try { + return toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string(); + } catch (IOException e) { + return super.toString(); + } + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java new file mode 100644 index 0000000000000..8a4461842a328 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java @@ -0,0 +1,328 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.apache.lucene.search.suggest.xdocument.CompletionQuery; +import org.apache.lucene.search.suggest.xdocument.ContextQuery; +import org.apache.lucene.search.suggest.xdocument.ContextSuggestField; +import org.apache.lucene.search.suggest.xdocument.TopSuggestDocs; +import org.apache.lucene.util.CharsRefBuilder; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.common.collect.ImmutableOpenIntMap; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.CompletionFieldMapper; +import org.elasticsearch.search.suggest.completion.CompletionSuggester; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionContext; + +import java.io.IOException; +import java.util.*; + +import static org.elasticsearch.search.suggest.completion.context.ContextMapping.*; + +/** + * ContextMappings indexes context-enabled suggestion fields + * and creates context queries for defined {@link ContextMapping}s + * for a {@link CompletionFieldMapper} + */ +public class ContextMappings implements ToXContent { + private final List contextMappings; + private final Map contextNameMap; + + private ContextMappings(List contextMappings) { + if (contextMappings.size() > 255) { + // we can support more, but max of 255 (1 byte) unique context types per suggest field + // seems reasonable? + throw new UnsupportedOperationException("Maximum of 10 context types are supported was: " + contextMappings.size()); + } + this.contextMappings = contextMappings; + contextNameMap = new HashMap<>(contextMappings.size()); + for (ContextMapping mapping : contextMappings) { + contextNameMap.put(mapping.name(), mapping); + } + } + + /** + * @return number of context mappings + * held by this instance + */ + public int size() { + return contextMappings.size(); + } + + /** + * Returns a context mapping by its name + */ + public ContextMapping get(String name) { + ContextMapping contextMapping = contextNameMap.get(name); + if (contextMapping == null) { + throw new IllegalArgumentException("Unknown context name[" + name + "], must be one of " + contextNameMap.size()); + } + return contextMapping; + } + + /** + * Adds a context-enabled field for all the defined mappings to document + * see {@link org.elasticsearch.search.suggest.completion.context.ContextMappings.TypedContextField} + */ + public void addFields(ParseContext.Document document, String name, String input, int weight, Map> contexts) { + for (int typeId = 0; typeId < contextMappings.size(); typeId++) { + ContextMapping mapping = contextMappings.get(typeId); + Set ctxs = contexts.get(mapping.name()); + if (ctxs == null) { + ctxs = new HashSet<>(); + } + document.add(new TypedContextField(name, typeId, input, weight, ctxs, document, mapping)); + } + } + + /** + * Field prepends context values with a suggestion + * Context values are associated with a type, denoted by + * a type id, which is prepended to the context value. + * + * Every defined context mapping yields a unique type id (index of the + * corresponding context mapping in the context mappings list) + * for all its context values + * + * The type, context and suggestion values are encoded as follows: + *

    + * TYPE_ID | CONTEXT_VALUE | CONTEXT_SEP | SUGGESTION_VALUE + *

    + * + * Field can also use values of other indexed fields as contexts + * at index time + */ + private static class TypedContextField extends ContextSuggestField { + private final int typeId; + private final Set contexts; + private final ParseContext.Document document; + private final ContextMapping mapping; + + public TypedContextField(String name, int typeId, String value, int weight, Set contexts, + ParseContext.Document document, ContextMapping mapping) { + super(name, value, weight); + this.typeId = typeId; + this.contexts = contexts; + this.document = document; + this.mapping = mapping; + } + + @Override + protected Iterable contexts() { + contexts.addAll(mapping.parseContext(document)); + final Iterator contextsIterator = contexts.iterator(); + final CharsRefBuilder scratch = new CharsRefBuilder(); + scratch.append(((char) typeId)); + return new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + @Override + public boolean hasNext() { + return contextsIterator.hasNext(); + } + + @Override + public CharSequence next() { + scratch.setLength(1); + scratch.append(contextsIterator.next()); + return scratch.toCharsRef(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove not supported"); + } + }; + } + }; + } + } + + /** + * Wraps a {@link CompletionQuery} with context queries, + * individual context mappings adds query contexts using + * {@link ContextMapping#addQueryContexts(ContextQuery, QueryContexts)}s + * + * @param query base completion query to wrap + * @param queryContexts a map of context mapping name and collected query contexts + * see {@link ContextMappingsParser#parseQueryContext(ContextMappings, XContentParser)} + * @return a context-enabled query + */ + public ContextQuery toContextQuery(CompletionQuery query, Map queryContexts) { + CharsRefBuilder scratch = new CharsRefBuilder(); + TypedContextQuery contextQuery = new TypedContextQuery(query, scratch); + for (int typeId = 0; typeId < contextMappings.size(); typeId++) { + ContextMapping mapping = contextMappings.get(typeId); + contextQuery.setTypeId(typeId); + QueryContexts queryContext = queryContexts.get(mapping.name()); + if (queryContext != null) { + if (queryContext.size() == 0) { + contextQuery.addAllContexts(); + } else { + mapping.addQueryContexts(contextQuery, queryContext); + } + } + } + return contextQuery; + } + + /** + * Wraps a Context query to prepend the context values + * with a type id + */ + private static class TypedContextQuery extends ContextQuery { + private final CharsRefBuilder scratch; + + public TypedContextQuery(CompletionQuery innerQuery, CharsRefBuilder scratch) { + super(innerQuery); + this.scratch = scratch; + } + + public void setTypeId(int typeId) { + scratch.clear(); + scratch.append(((char) typeId)); + } + + @Override + public final void addContext(CharSequence context, float boost, boolean exact) { + scratch.setLength(1); + if (context != null) { + scratch.append(context); + } + super.addContext(scratch.toCharsRef(), boost, exact); + } + + @Override + public final void addAllContexts() { + addContext(null, 1, false); + } + } + + /** + * Maps an output context list to a map of context mapping names and their values + * + * see {@link org.elasticsearch.search.suggest.completion.context.ContextMappings.TypedContextField} + * @param context from {@link TopSuggestDocs.SuggestScoreDoc#context} + * @return a map of context names and their values + * + */ + public Map.Entry getNamedContext(CharSequence context) { + int typeId = context.charAt(0); + assert typeId < contextMappings.size() : "Returned context has invalid type"; + ContextMapping mapping = contextMappings.get(typeId); + return new AbstractMap.SimpleEntry<>(mapping.name(), context.subSequence(1, context.length())); + } + + /** + * Loads {@link ContextMappings} from configuration + * + * Expected configuration: + * List of maps representing {@link ContextMapping} + * [{"name": .., "type": .., ..}, {..}] + * + */ + public static ContextMappings load(Object configuration, Version indexVersionCreated) throws ElasticsearchParseException { + final List contextMappings; + if (configuration instanceof List) { + contextMappings = new ArrayList<>(); + List configurations = (List)configuration; + for (Object contextConfig : configurations) { + contextMappings.add(load((Map) contextConfig, indexVersionCreated)); + } + if (contextMappings.size() == 0) { + throw new ElasticsearchParseException("expected at least one context mapping"); + } + } else if (configuration instanceof Map) { + contextMappings = Collections.singletonList(load(((Map) configuration), indexVersionCreated)); + } else { + throw new ElasticsearchParseException("expected a list or an entry of context mapping"); + } + return new ContextMappings(contextMappings); + } + + private static ContextMapping load(Map contextConfig, Version indexVersionCreated) { + String name = extractRequiredValue(contextConfig, FIELD_NAME); + String type = extractRequiredValue(contextConfig, FIELD_TYPE); + final ContextMapping contextMapping; + switch (Type.fromString(type)) { + case CATEGORY: + contextMapping = CategoryContextMapping.load(name, contextConfig); + break; + case GEO: + contextMapping = GeoContextMapping.load(name, contextConfig); + break; + default: + throw new ElasticsearchParseException("unknown context type[" + type + "]"); + } + DocumentMapperParser.checkNoRemainingFields(name, contextConfig, indexVersionCreated); + return contextMapping; + } + + private static String extractRequiredValue(Map contextConfig, String paramName) { + final Object paramValue = contextConfig.get(paramName); + if (paramValue == null) { + throw new ElasticsearchParseException("missing [" + paramName + "] in context mapping"); + } + contextConfig.remove(paramName); + return paramValue.toString(); + } + + /** + * Writes a list of objects specified by the defined {@link ContextMapping}s + * + * see {@link ContextMapping#toXContent(XContentBuilder, Params)} + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (ContextMapping contextMapping : contextMappings) { + builder.startObject(); + contextMapping.toXContent(builder, params); + builder.endObject(); + } + return builder; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + for (ContextMapping contextMapping : contextMappings) { + result = prime * result + contextMapping.hashCode(); + } + return result; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || (obj instanceof ContextMappings) == false) { + return false; + } + ContextMappings other = ((ContextMappings) obj); + return contextMappings.equals(other.contextMappings); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappingsParser.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappingsParser.java new file mode 100644 index 0000000000000..e57e966656963 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappingsParser.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.search.suggest.completion.CompletionSuggester; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionContext; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Parses index-time context values and query contexts + * for all defined context mappings in + * {@link ContextMappings} + */ +public class ContextMappingsParser { + + /** + * Parses query contexts for all the defined contexts + * + * Used in {@link CompletionSuggester#toQuery(CompletionSuggestionContext)} + *
    + * Expected Input: + *
      + * 
    • {"NAME": <QUERY_CONTEXTS>, ..}
+     *
    + * see specific {@link ContextMapping#parseQueryContext(String, XContentParser)} implementation + * for QUERY_CONTEXTS + * NAME refers to the name of context mapping + */ + public static Map parseQueryContext(ContextMappings contextMappings, XContentParser parser) throws IOException { + Map queryContextsMap = new HashMap<>(contextMappings.size()); + assert parser.currentToken() == XContentParser.Token.START_OBJECT; + XContentParser.Token token; + String currentFieldName; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + final ContextMapping mapping = contextMappings.get(currentFieldName); + queryContextsMap.put(currentFieldName, mapping.parseQueryContext(currentFieldName, parser)); + } + + } + return queryContextsMap; + } + + /** + * Parses index-time contexts for all the defined context mappings + * + * Expected input : + *
      + *
    • Array: 
      [{<NAME>: <CONTEXTS>}, ]
      • + *
    • Object: 
      {<NAME>: <CONTEXTS>}
      • + *
    + * + * see specific {@link ContextMapping#parseContext(ParseContext, XContentParser)} implementations + * for CONTEXTS. + * NAME refers to the name of context mapping + */ + public static Map> parseContext(ContextMappings contextMappings, ParseContext parseContext, XContentParser parser) throws IOException { + Map> contextMap = new HashMap<>(contextMappings.size()); + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.START_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + ContextMapping mapping = contextMappings.get(currentFieldName); + Set contexts = contextMap.get(currentFieldName); + if (contexts == null) { + contexts = new HashSet<>(); + } + contexts.addAll(mapping.parseContext(parseContext, parser)); + contextMap.put(currentFieldName, contexts); + } + } + } + } else if (token == XContentParser.Token.START_OBJECT) { + ContextMapping contextMapping = null; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + contextMapping = contextMappings.get(currentFieldName); + } else if (token == XContentParser.Token.VALUE_STRING || token == XContentParser.Token.START_ARRAY || token == XContentParser.Token.START_OBJECT) { + assert currentFieldName != null; + Set contexts = contextMap.get(currentFieldName); + if (contexts == null) { + contexts = new HashSet<>(); + } + contexts.addAll(contextMapping.parseContext(parseContext, parser)); + contextMap.put(currentFieldName, contexts); + } + } + } + return contextMap; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java new file mode 100644 index 0000000000000..176c0e45f3a03 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoContextMapping.java @@ -0,0 +1,502 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.suggest.xdocument.ContextQuery; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.geo.GeoHashUtils; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; + +import java.io.IOException; +import java.util.*; + +/** + * A {@link ContextMapping} that uses a geo location/area as a + * criteria. + * The suggestions can be boosted and/or filtered depending on + * whether it falls within an area, represented by a query geo hash + * with a specified precision + * + * {@link GeoQueryContext} defines the options for constructing + * a unit of query context for this context type + */ +public class GeoContextMapping extends ContextMapping { + + public static final String FIELD_PRECISION = "precision"; + public static final String FIELD_FIELDNAME = "path"; + + public static final int DEFAULT_PRECISION = 6; + + static final String CONTEXT_VALUE = "context"; + static final String CONTEXT_BOOST = "boost"; + static final String CONTEXT_PRECISION = "precision"; + static final String CONTEXT_NEIGHBOURS = "neighbours"; + + private final int precision; + private final String fieldName; + + private GeoContextMapping(String name, String fieldName, int precision) { + super(Type.GEO, name); + this.precision = precision; + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + public int getPrecision() { + return precision; + } + + protected static GeoContextMapping load(String name, Map config) { + final GeoContextMapping.Builder builder = new GeoContextMapping.Builder(name); + + if (config != null) { + final Object configPrecision = config.get(FIELD_PRECISION); + if (configPrecision != null) { + if (configPrecision instanceof Integer) { + builder.precision((Integer) configPrecision); + } else if (configPrecision instanceof Long) { + builder.precision((Long) configPrecision); + } else if (configPrecision instanceof Double) { + builder.precision((Double) configPrecision); + } else if (configPrecision instanceof Float) { + builder.precision((Float) configPrecision); + } else { + builder.precision(configPrecision.toString()); + } + config.remove(FIELD_PRECISION); + } + + final Object fieldName = config.get(FIELD_FIELDNAME); + if (fieldName != null) { + builder.field(fieldName.toString()); + config.remove(FIELD_FIELDNAME); + } + } + return builder.build(); + } + + @Override + protected XContentBuilder toInnerXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FIELD_PRECISION, precision); + if (fieldName != null) { + builder.field(FIELD_FIELDNAME, fieldName); + } + return builder; + } + + /** + * Parse a set of {@link CharSequence} contexts at index-time. + * Acceptable formats: + * + *
      + *
    • Array: 
      [<GEO POINT>, ..]
      • + *
    • String/Object/Array: 
      "GEO POINT"
      • + *
    + * + * see {@link GeoUtils#parseGeoPoint(String, GeoPoint)} for GEO POINT + */ + @Override + public Set parseContext(ParseContext parseContext, XContentParser parser) throws IOException, ElasticsearchParseException { + if (fieldName != null) { + FieldMapper mapper = parseContext.docMapper().mappers().getMapper(fieldName); + if (!(mapper instanceof GeoPointFieldMapper)) { + throw new ElasticsearchParseException("referenced field must be mapped to geo_point"); + } + } + + final Set contexts = new HashSet<>(); + Token token = parser.currentToken(); + if (token == Token.START_ARRAY) { + token = parser.nextToken(); + // Test if value is a single point in [lon, lat] format + if (token == Token.VALUE_NUMBER) { + double lon = parser.doubleValue(); + if (parser.nextToken() == Token.VALUE_NUMBER) { + double lat = parser.doubleValue(); + if (parser.nextToken() == Token.END_ARRAY) { + contexts.add(GeoHashUtils.encode(lat, lon, precision)); + } else { + throw new ElasticsearchParseException("only two values expected"); + } + } else { + throw new ElasticsearchParseException("latitude must be a numeric value"); + } + } else { + while (token != Token.END_ARRAY) { + GeoPoint point = GeoUtils.parseGeoPoint(parser); + contexts.add(GeoHashUtils.encode(point.getLat(), point.getLon(), precision)); + token = parser.nextToken(); + } + } + } else { + // or a single location + GeoPoint point = GeoUtils.parseGeoPoint(parser); + contexts.add(GeoHashUtils.encode(point.getLat(), point.getLon(), precision)); + } + return contexts; + } + + @Override + public Set parseContext(Document document) { + final Set geohashes = new HashSet<>(); + + if (fieldName != null) { + IndexableField[] fields = document.getFields(fieldName); + GeoPoint spare = new GeoPoint(); + if (fields.length == 0) { + IndexableField[] lonFields = document.getFields(fieldName + ".lon"); + IndexableField[] latFields = document.getFields(fieldName + ".lat"); + if (lonFields.length > 0 && latFields.length > 0) { + for (int i = 0; i < lonFields.length; i++) { + IndexableField lonField = lonFields[i]; + IndexableField latField = latFields[i]; + assert lonField.fieldType().docValuesType() == latField.fieldType().docValuesType(); + // we write doc values fields differently: one field for all values, so we need to only care about indexed fields + if (lonField.fieldType().docValuesType() == DocValuesType.NONE) { + spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue()); + geohashes.add(GeoHashUtils.encode(spare.getLat(), spare.getLon(), precision)); + } + } + } + } else { + for (IndexableField field : fields) { + spare.resetFromString(field.stringValue()); + geohashes.add(spare.geohash()); + } + } + } + + Set locations = new HashSet<>(); + for (CharSequence geohash : geohashes) { + int precision = Math.min(this.precision, geohash.length()); + CharSequence truncatedGeohash = geohash.subSequence(0, precision); + locations.add(truncatedGeohash); + } + return locations; + } + + /** + * Parse a {@link QueryContexts} + * using parser. A QueryContexts accepts one of the following forms: + * + *
      + *
    • Object: GeoQueryContext
      • + *
    • String: GeoQueryContext value with boost=1 precision=PRECISION neighbours=[PRECISION]
      • + *
    • Array: 
      [GeoQueryContext, ..]
      • + *
    + * + * A GeoQueryContext has one of the following forms: + *
      + *
    • Object: + *
        + * 
      • GEO POINT
        • + * 
      • {"lat": <double>, "lon": <double>, "precision": <int>, "neighbours": <[int, ..]>}
        • + * 
      • {"context": <string>, "boost": <int>, "precision": <int>, "neighbours": <[int, ..]>}
        • + * 
      • {"context": <GEO POINT>, "boost": <int>, "precision": <int>, "neighbours": <[int, ..]>}
        • + *
      + *
    • String: 
      GEO POINT
      • + *
    + * see {@link GeoUtils#parseGeoPoint(String, GeoPoint)} for GEO POINT + */ + @Override + public QueryContexts parseQueryContext(String name, XContentParser parser) throws IOException, ElasticsearchParseException { + QueryContexts queryContexts = new QueryContexts<>(name); + Token token = parser.nextToken(); + if (token == Token.START_OBJECT || token == Token.VALUE_STRING) { + GeoQueryContext current = innerParseQueryContext(parser); + if (current != null) { + queryContexts.add(current); + } + } else if (token == Token.START_ARRAY) { + while (parser.nextToken() != Token.END_ARRAY) { + GeoQueryContext current = innerParseQueryContext(parser); + if (current != null) { + queryContexts.add(current); + } + } + } + return queryContexts; + } + + private GeoQueryContext innerParseQueryContext(XContentParser parser) throws IOException, ElasticsearchParseException { + Token token = parser.currentToken(); + if (token == Token.VALUE_STRING) { + return new GeoQueryContext(GeoUtils.parseGeoPoint(parser), 1, precision, precision); + } else if (token == Token.START_OBJECT) { + String currentFieldName = null; + GeoPoint point = null; + double lat = Double.NaN; + double lon = Double.NaN; + int precision = this.precision; + List neighbours = new ArrayList<>(); + int boost = 1; + while ((token = parser.nextToken()) != Token.END_OBJECT) { + if (token == Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == Token.VALUE_STRING) { + if ("lat".equals(currentFieldName)) { + if (point == null) { + lat = parser.doubleValue(true); + } else { + throw new ElasticsearchParseException("only lat/lon is allowed"); + } + } else if ("lon".equals(currentFieldName)) { + if (point == null) { + lon = parser.doubleValue(true); + } else { + throw new ElasticsearchParseException("only lat/lon is allowed"); + } + } else if (CONTEXT_VALUE.equals(currentFieldName)) { + point = GeoUtils.parseGeoPoint(parser); + } else if (CONTEXT_BOOST.equals(currentFieldName)) { + Number number; + try { + number = Long.parseLong(parser.text()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Boost must be a string representing a numeric value, but was [" + parser.text() + "]"); + } + boost = number.intValue(); + } else if (CONTEXT_NEIGHBOURS.equals(currentFieldName)) { + neighbours.add(GeoUtils.geoHashLevelsForPrecision(parser.text())); + } else if (CONTEXT_PRECISION.equals(currentFieldName)) { + precision = GeoUtils.geoHashLevelsForPrecision(parser.text()); + } else { + throw new ElasticsearchParseException("unknown field [" + currentFieldName + "] for string value"); + } + } else if (token == Token.VALUE_NUMBER) { + if ("lat".equals(currentFieldName)) { + if (point == null) { + lat = parser.doubleValue(true); + } else { + throw new ElasticsearchParseException("only lat/lon is allowed"); + } + } else if ("lon".equals(currentFieldName)) { + if (point == null) { + lon = parser.doubleValue(true); + } else { + throw new ElasticsearchParseException("only lat/lon is allowed"); + } + } else if (CONTEXT_NEIGHBOURS.equals(currentFieldName)) { + XContentParser.NumberType numberType = parser.numberType(); + if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) { + neighbours.add(parser.intValue()); + } else { + neighbours.add(GeoUtils.geoHashLevelsForPrecision(parser.doubleValue())); + } + } else if (CONTEXT_PRECISION.equals(currentFieldName)) { + XContentParser.NumberType numberType = parser.numberType(); + if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) { + precision = parser.intValue(); + } else { + precision = GeoUtils.geoHashLevelsForPrecision(parser.doubleValue()); + } + } else if (CONTEXT_BOOST.equals(currentFieldName)) { + XContentParser.NumberType numberType = parser.numberType(); + Number number = parser.numberValue(); + if (numberType == XContentParser.NumberType.INT) { + boost = number.intValue(); + } else { + throw new ElasticsearchParseException("boost must be in the interval [0..2147483647], but was [" + number.longValue() + "]"); + } + } else { + throw new ElasticsearchParseException("unknown field [" + currentFieldName + "] for numeric value"); + } + } else if (token == Token.START_OBJECT) { + if (CONTEXT_VALUE.equals(currentFieldName)) { + point = GeoUtils.parseGeoPoint(parser); + } else { + throw new ElasticsearchParseException("unknown field [" + currentFieldName + "] for object value"); + } + } else if (token == Token.START_ARRAY) { + if (CONTEXT_NEIGHBOURS.equals(currentFieldName)) { + while ((token = parser.nextToken()) != Token.END_ARRAY) { + if (token == Token.VALUE_STRING || token == Token.VALUE_NUMBER) { + neighbours.add(parser.intValue(true)); + } else { + throw new ElasticsearchParseException("neighbours accept string/numbers"); + } + } + } else { + throw new ElasticsearchParseException("unknown field [" + currentFieldName + "] for array value"); + } + } + } + if (point == null) { + if (Double.isNaN(lat) == false && Double.isNaN(lon) == false) { + point = new GeoPoint(lat, lon); + } else { + throw new ElasticsearchParseException("no context value"); + } + } + if (neighbours.size() > 0) { + final int[] neighbourValues = new int[neighbours.size()]; + for (int i = 0; i < neighbours.size(); i++) { + neighbourValues[i] = neighbours.get(i); + } + return new GeoQueryContext(point, boost, precision, neighbourValues); + } else { + return new GeoQueryContext(point, boost, precision, precision); + } + } else { + throw new ElasticsearchParseException("expected string or object"); + } + } + + + @Override + public void addQueryContexts(ContextQuery query, QueryContexts queryContexts) { + for (GeoQueryContext queryContext : queryContexts) { + int precision = Math.min(this.precision, queryContext.geoHash.length()); + String truncatedGeohash = queryContext.geoHash.toString().substring(0, precision); + query.addContext(truncatedGeohash, queryContext.boost, false); + for (int neighboursPrecision : queryContext.neighbours) { + int neighbourPrecision = Math.min(neighboursPrecision, truncatedGeohash.length()); + String neighbourGeohash = truncatedGeohash.substring(0, neighbourPrecision); + Collection locations = new HashSet<>(); + GeoHashUtils.addNeighbors(neighbourGeohash, precision, locations); + for (String location : locations) { + query.addContext(location, queryContext.boost, false); + } + } + } + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((fieldName == null) ? 0 : fieldName.hashCode()); + result = prime * result + precision; + return result; + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + if (obj instanceof GeoContextMapping) { + GeoContextMapping other = (GeoContextMapping) obj; + if (fieldName == null) { + if (other.fieldName != null) { + return false; + } + } else if (!fieldName.equals(other.fieldName)) { + return false; + } else if (precision != other.precision) { + return false; + } + return true; + } + } + return false; + } + + public static class Builder extends ContextBuilder { + + private int precision = DEFAULT_PRECISION; + private String fieldName = null; + + protected Builder(String name) { + super(name); + } + + /** + * Set the precision use o make suggestions + * + * @param precision + * precision as distance with {@link DistanceUnit}. Default: + * meters + * @return this + */ + public Builder precision(String precision) { + return precision(DistanceUnit.parse(precision, DistanceUnit.METERS, DistanceUnit.METERS)); + } + + /** + * Set the precision use o make suggestions + * + * @param precision + * precision value + * @param unit + * {@link DistanceUnit} to use + * @return this + */ + public Builder precision(double precision, DistanceUnit unit) { + return precision(unit.toMeters(precision)); + } + + /** + * Set the precision use o make suggestions + * + * @param meters + * precision as distance in meters + * @return this + */ + public Builder precision(double meters) { + int level = GeoUtils.geoHashLevelsForPrecision(meters); + // Ceiling precision: we might return more results + if (GeoUtils.geoHashCellSize(level) < meters) { + level = Math.max(1, level - 1); + } + return precision(level); + } + + /** + * Set the precision use o make suggestions + * + * @param level + * maximum length of geohashes + * @return this + */ + public Builder precision(int level) { + this.precision = level; + return this; + } + + /** + * Set the name of the field containing a geolocation to use + * @param fieldName name of the field + * @return this + */ + public Builder field(String fieldName) { + this.fieldName = fieldName; + return this; + } + + @Override + public GeoContextMapping build() { + return new GeoContextMapping(name, fieldName, precision); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java new file mode 100644 index 0000000000000..8a3f76886c0dd --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java @@ -0,0 +1,101 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion.context; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +import static org.elasticsearch.search.suggest.completion.context.GeoContextMapping.*; + +/** + * Defines the query context for {@link GeoContextMapping} + */ +public class GeoQueryContext implements ToXContent { + public final CharSequence geoHash; + public final int precision; + public final int boost; + public final int[] neighbours; + + /** + * Creates a query context for a given geo point with a boost of 1 + * and a precision of {@value GeoContextMapping#DEFAULT_PRECISION} + */ + public GeoQueryContext(GeoPoint geoPoint) { + this(geoPoint.geohash()); + } + + /** + * Creates a query context for a given geo point with a + * provided boost + */ + public GeoQueryContext(GeoPoint geoPoint, int boost) { + this(geoPoint.geohash(), boost); + } + + /** + * Creates a query context with a given geo hash with a boost of 1 + * and a precision of {@value GeoContextMapping#DEFAULT_PRECISION} + */ + public GeoQueryContext(CharSequence geoHash) { + this(geoHash, 1); + } + + /** + * Creates a query context for a given geo hash with a + * provided boost + */ + public GeoQueryContext(CharSequence geoHash, int boost) { + this(geoHash, boost, DEFAULT_PRECISION); + } + + /** + * Creates a query context for a geo point with + * a provided boost and enables generating neighbours + * at specified precisions + */ + public GeoQueryContext(GeoPoint geoPoint, int boost, int precision, int... neighbours) { + this(geoPoint.geohash(), boost, precision, neighbours); + } + + /** + * Creates a query context for a geo hash with + * a provided boost and enables generating neighbours + * at specified precisions + */ + public GeoQueryContext(CharSequence geoHash, int boost, int precision, int... neighbours) { + this.geoHash = geoHash; + this.boost = boost; + this.precision = precision; + this.neighbours = neighbours; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONTEXT_VALUE, geoHash); + builder.field(CONTEXT_BOOST, boost); + builder.field(CONTEXT_NEIGHBOURS, neighbours); + builder.endObject(); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/AnalyzingCompletionLookupProvider.java similarity index 96% rename from core/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/AnalyzingCompletionLookupProvider.java index 4ee79025adf95..b1b0f9a4be7e7 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/AnalyzingCompletionLookupProvider.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import com.carrotsearch.hppc.ObjectLongHashMap; @@ -47,10 +47,9 @@ import org.apache.lucene.util.fst.PositiveIntOutputs; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping; import java.io.IOException; import java.util.Collection; @@ -60,7 +59,7 @@ import java.util.Set; import java.util.TreeMap; -public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider { +public class AnalyzingCompletionLookupProvider extends Completion090PostingsFormat.CompletionLookupProvider { // for serialization public static final int SERIALIZE_PRESERVE_SEPARATORS = 1; @@ -208,7 +207,7 @@ public void write(Fields fields) throws IOException { @Override - public LookupFactory load(IndexInput input) throws IOException { + public Completion090PostingsFormat.LookupFactory load(IndexInput input) throws IOException { long sizeInBytes = 0; int version = CodecUtil.checkHeader(input, CODEC_NAME, CODEC_VERSION_START, CODEC_VERSION_LATEST); if (version >= CODEC_VERSION_CHECKSUMS) { @@ -263,9 +262,9 @@ public LookupFactory load(IndexInput input) throws IOException { lookupMap.put(entry.getValue(), holder); } final long ramBytesUsed = sizeInBytes; - return new LookupFactory() { + return new Completion090PostingsFormat.LookupFactory() { @Override - public Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext) { + public Lookup getLookup(OldCompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext) { AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(fieldType.names().indexName()); if (analyzingSuggestHolder == null) { return null; @@ -273,7 +272,7 @@ public Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, Com int flags = analyzingSuggestHolder.getPreserveSeparator() ? XAnalyzingSuggester.PRESERVE_SEP : 0; final XAnalyzingSuggester suggester; - final Automaton queryPrefix = fieldType.requiresContext() ? ContextQuery.toAutomaton(analyzingSuggestHolder.getPreserveSeparator(), suggestionContext.getContextQueries()) : null; + final Automaton queryPrefix = fieldType.requiresContext() ? ContextMapping.ContextQuery.toAutomaton(analyzingSuggestHolder.getPreserveSeparator(), suggestionContext.getContextQueries()) : null; if (suggestionContext.isFuzzy()) { suggester = new XFuzzySuggester(fieldType.indexAnalyzer(), queryPrefix, fieldType.searchAnalyzer(), flags, diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/Completion090PostingsFormat.java similarity index 96% rename from core/src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/Completion090PostingsFormat.java index bbb3340c68539..ce5707b15a808 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/Completion090PostingsFormat.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; @@ -46,8 +46,8 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream.ToFiniteStrings; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; +import org.elasticsearch.search.suggest.completion.CompletionStats; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -260,7 +260,7 @@ public CompletionTerms(Terms delegate, LookupFactory lookup) { this.lookup = lookup; } - public Lookup getLookup(CompletionFieldMapper.CompletionFieldType mapper, CompletionSuggestionContext suggestionContext) { + public Lookup getLookup(OldCompletionFieldMapper.CompletionFieldType mapper, CompletionSuggestionContext suggestionContext) { return lookup.getLookup(mapper, suggestionContext); } @@ -269,7 +269,7 @@ public CompletionStats stats(String ... fields) { } } - public static abstract class CompletionLookupProvider implements PayloadProcessor, ToFiniteStrings { + public static abstract class CompletionLookupProvider implements PayloadProcessor, CompletionTokenStream.ToFiniteStrings { public static final char UNIT_SEPARATOR = '\u001f'; @@ -340,7 +340,7 @@ public CompletionStats completionStats(IndexReader indexReader, String ... field } public static abstract class LookupFactory implements Accountable { - public abstract Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext); + public abstract Lookup getLookup(OldCompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext); public abstract CompletionStats stats(String ... fields); abstract AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder getAnalyzingSuggestHolder(MappedFieldType fieldType); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggester.java new file mode 100644 index 0000000000000..5a1a27b411e38 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggester.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest.completion.old; + +import com.google.common.collect.Maps; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.text.StringText; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestContextParser; +import org.elasticsearch.search.suggest.Suggester; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +public class CompletionSuggester extends Suggester { + + private static final ScoreComparator scoreComparator = new ScoreComparator(); + + + @Override + protected Suggest.Suggestion> innerExecute(String name, + CompletionSuggestionContext suggestionContext, IndexSearcher searcher, CharsRefBuilder spare) throws IOException { + if (suggestionContext.fieldType() == null) { + throw new ElasticsearchException("Field [" + suggestionContext.getField() + "] is not a completion suggest field"); + } + final IndexReader indexReader = searcher.getIndexReader(); + CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize()); + spare.copyUTF8Bytes(suggestionContext.getText()); + + CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new StringText(spare.toString()), 0, spare.length()); + completionSuggestion.addTerm(completionSuggestEntry); + + String fieldName = suggestionContext.getField(); + Map results = Maps.newHashMapWithExpectedSize(indexReader.leaves().size() * suggestionContext.getSize()); + for (LeafReaderContext atomicReaderContext : indexReader.leaves()) { + LeafReader atomicReader = atomicReaderContext.reader(); + Terms terms = atomicReader.fields().terms(fieldName); + if (terms instanceof Completion090PostingsFormat.CompletionTerms) { + final Completion090PostingsFormat.CompletionTerms lookupTerms = (Completion090PostingsFormat.CompletionTerms) terms; + final Lookup lookup = lookupTerms.getLookup(suggestionContext.fieldType(), suggestionContext); + if (lookup == null) { + // we don't have a lookup for this segment.. this might be possible if a merge dropped all + // docs from the segment that had a value in this segment. + continue; + } + List lookupResults = lookup.lookup(spare.get(), false, suggestionContext.getSize()); + for (Lookup.LookupResult res : lookupResults) { + + final String key = res.key.toString(); + final float score = res.value; + final CompletionSuggestion.Entry.Option value = results.get(key); + if (value == null) { + final CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(new StringText(key), score, res.payload == null ? null + : new BytesArray(res.payload)); + results.put(key, option); + } else if (value.getScore() < score) { + value.setScore(score); + value.setPayload(res.payload == null ? null : new BytesArray(res.payload)); + } + } + } + } + final List options = new ArrayList<>(results.values()); + CollectionUtil.introSort(options, scoreComparator); + + int optionCount = Math.min(suggestionContext.getSize(), options.size()); + for (int i = 0 ; i < optionCount ; i++) { + completionSuggestEntry.addOption(options.get(i)); + } + + return completionSuggestion; + } + + @Override + public String[] names() { + return new String[] { "completion_old" }; + } + + @Override + public SuggestContextParser getContextParser() { + throw new IllegalStateException("This method should not be called"); + } + + public static class ScoreComparator implements Comparator { + @Override + public int compare(CompletionSuggestion.Entry.Option o1, CompletionSuggestion.Entry.Option o2) { + return Float.compare(o2.getScore(), o1.getScore()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestion.java new file mode 100644 index 0000000000000..40669cb49843a --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestion.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest.completion.old; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.search.suggest.Suggest; + +import java.io.IOException; +import java.util.Map; + +/** + * + */ +public class CompletionSuggestion extends Suggest.Suggestion { + + public static final int TYPE = 2; + + public CompletionSuggestion() { + } + + public CompletionSuggestion(String name, int size) { + super(name, size); + } + + @Override + public int getType() { + return TYPE; + } + + @Override + protected Entry newEntry() { + return new Entry(); + } + + public static class Entry extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry { + + public Entry(Text text, int offset, int length) { + super(text, offset, length); + } + + protected Entry() { + super(); + } + + @Override + protected Option newOption() { + return new Option(); + } + + public static class Option extends org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option { + private BytesReference payload; + + public Option(Text text, float score, BytesReference payload) { + super(text, score); + this.payload = payload; + } + + + protected Option() { + super(); + } + + public void setPayload(BytesReference payload) { + this.payload = payload; + } + + public BytesReference getPayload() { + return payload; + } + + public String getPayloadAsString() { + return payload.toUtf8(); + } + + public long getPayloadAsLong() { + return Long.parseLong(payload.toUtf8()); + } + + public double getPayloadAsDouble() { + return Double.parseDouble(payload.toUtf8()); + } + + public Map getPayloadAsMap() { + return XContentHelper.convertToMap(payload, false).v2(); + } + + @Override + public void setScore(float score) { + super.setScore(score); + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + super.innerToXContent(builder, params); + if (payload != null && payload.length() > 0) { + builder.rawField("payload", payload); + } + return builder; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + payload = in.readBytesReference(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBytesReference(payload); + } + } + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionBuilder.java new file mode 100644 index 0000000000000..4b16accc53cfe --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionBuilder.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest.completion.old; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.suggest.SuggestBuilder; + +import java.io.IOException; + +/** + * Defines a suggest command based on a prefix, typically to provide "auto-complete" functionality + * for users as they type search terms. The implementation of the completion service uses FSTs that + * are created at index-time and so must be defined in the mapping with the type "completion" before + * indexing. + */ +public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilder { + + public CompletionSuggestionBuilder(String name) { + super(name, "completion"); + } + + @Override + protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionContext.java new file mode 100644 index 0000000000000..0f5144c759de1 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionContext.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest.completion.old; + +import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; +import org.elasticsearch.search.suggest.Suggester; +import org.elasticsearch.search.suggest.SuggestionSearchContext; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping.ContextQuery; + +import java.util.Collections; +import java.util.List; + +/** + * + */ +public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext { + + private OldCompletionFieldMapper.CompletionFieldType fieldType; + private int fuzzyEditDistance = XFuzzySuggester.DEFAULT_MAX_EDITS; + private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS; + private int fuzzyMinLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH; + private int fuzzyPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX; + private boolean fuzzy = false; + private boolean fuzzyUnicodeAware = XFuzzySuggester.DEFAULT_UNICODE_AWARE; + private List contextQueries = Collections.emptyList(); + + public CompletionSuggestionContext(Suggester suggester) { + super(suggester); + } + + public OldCompletionFieldMapper.CompletionFieldType fieldType() { + return this.fieldType; + } + + public void fieldType(OldCompletionFieldMapper.CompletionFieldType fieldType) { + this.fieldType = fieldType; + } + + public void setFuzzyEditDistance(int fuzzyEditDistance) { + this.fuzzyEditDistance = fuzzyEditDistance; + } + + public int getFuzzyEditDistance() { + return fuzzyEditDistance; + } + + public void setFuzzyTranspositions(boolean fuzzyTranspositions) { + this.fuzzyTranspositions = fuzzyTranspositions; + } + + public boolean isFuzzyTranspositions() { + return fuzzyTranspositions; + } + + public void setFuzzyMinLength(int fuzzyMinPrefixLength) { + this.fuzzyMinLength = fuzzyMinPrefixLength; + } + + public int getFuzzyMinLength() { + return fuzzyMinLength; + } + + public void setFuzzyPrefixLength(int fuzzyNonPrefixLength) { + this.fuzzyPrefixLength = fuzzyNonPrefixLength; + } + + public int getFuzzyPrefixLength() { + return fuzzyPrefixLength; + } + + public void setFuzzy(boolean fuzzy) { + this.fuzzy = fuzzy; + } + + public boolean isFuzzy() { + return fuzzy; + } + + public void setFuzzyUnicodeAware(boolean fuzzyUnicodeAware) { + this.fuzzyUnicodeAware = fuzzyUnicodeAware; + } + + public boolean isFuzzyUnicodeAware() { + return fuzzyUnicodeAware; + } + + public void setContextQuery(List queries) { + this.contextQueries = queries; + } + + public List getContextQueries() { + return this.contextQueries; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionFuzzyBuilder.java similarity index 98% rename from core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionFuzzyBuilder.java index de6bf1365d96c..09852b8a70eec 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionSuggestionFuzzyBuilder.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester; import org.elasticsearch.common.unit.Fuzziness; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionTokenStream.java similarity index 99% rename from core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionTokenStream.java index 103fd0dcf0a88..604221a4bee2f 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/CompletionTokenStream.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/PayloadProcessor.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/PayloadProcessor.java similarity index 95% rename from core/src/main/java/org/elasticsearch/search/suggest/completion/PayloadProcessor.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/PayloadProcessor.java index 544d9052a0ed2..d69c422e48f29 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/PayloadProcessor.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/PayloadProcessor.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/CategoryContextMapping.java similarity index 99% rename from core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/CategoryContextMapping.java index 975b42a0f013b..7a2d04e93e94f 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/CategoryContextMapping.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/CategoryContextMapping.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.context; +package org.elasticsearch.search.suggest.completion.old.context; import com.google.common.base.Joiner; import com.google.common.collect.Iterables; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/ContextBuilder.java similarity index 98% rename from core/src/main/java/org/elasticsearch/search/suggest/context/ContextBuilder.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/ContextBuilder.java index 0a1ea7eea8828..ab352a70b0283 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/ContextBuilder.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.context; +package org.elasticsearch.search.suggest.completion.old.context; import com.google.common.collect.Maps; import org.elasticsearch.ElasticsearchParseException; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/ContextMapping.java similarity index 99% rename from core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/ContextMapping.java index 43956c99dc9c7..b362121930d78 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/ContextMapping.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/ContextMapping.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.context; +package org.elasticsearch.search.suggest.completion.old.context; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; diff --git a/core/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/GeolocationContextMapping.java similarity index 99% rename from core/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java rename to core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/GeolocationContextMapping.java index 22975ff83f35e..f3da9db0fc5c1 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/old/context/GeolocationContextMapping.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.context; +package org.elasticsearch.search.suggest.completion.old.context; import com.carrotsearch.hppc.IntHashSet; import com.google.common.collect.Lists; diff --git a/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat index 52134e4fc82ad..06dca316c3228 100644 --- a/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat +++ b/core/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat @@ -1,3 +1,4 @@ org.elasticsearch.index.codec.postingsformat.Elasticsearch090PostingsFormat -org.elasticsearch.search.suggest.completion.Completion090PostingsFormat +org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat +org.apache.lucene.search.suggest.xdocument.Completion50PostingsFormat \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index 15242ab6fb9a3..5f0bbf3759291 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -904,8 +904,7 @@ public void testUngeneratedFieldsThatAreNeverStored() throws IOException { " \"input\": [\n" + " \"Nevermind\",\n" + " \"Nirvana\"\n" + - " ],\n" + - " \"output\": \"Nirvana - Nevermind\"\n" + + " ]\n" + " }\n" + "}"; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java b/core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java index 8e88cbb3d3c16..7ee79fbe60e5d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java @@ -22,16 +22,20 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.Version; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.groovy.GroovyScriptEngineService; import org.elasticsearch.search.suggest.SuggestBuilders; -import org.elasticsearch.search.suggest.completion.CompletionSuggestion; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestion; +import org.elasticsearch.test.VersionUtils; import org.junit.Test; import java.io.IOException; @@ -87,6 +91,7 @@ public void getTransformed() throws Exception { // ever fix the completion suggester to reencode the payloads then we can remove this test. @Test public void contextSuggestPayloadTransformed() throws Exception { + Version PRE2X_VERSION = VersionUtils.randomVersionBetween(getRandom(), Version.V_1_0_0, Version.V_1_7_0); XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); builder.startObject("properties"); builder.startObject("suggest").field("type", "completion").field("payloads", true).endObject(); @@ -95,14 +100,14 @@ public void contextSuggestPayloadTransformed() throws Exception { builder.field("script", "ctx._source.suggest = ['input': ctx._source.text];ctx._source.suggest.payload = ['display': ctx._source.text, 'display_detail': 'on the fly']"); builder.field("lang", GroovyScriptEngineService.NAME); builder.endObject(); - assertAcked(client().admin().indices().prepareCreate("test").addMapping("test", builder)); + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("test", builder)); // Payload is stored using original source format (json, smile, yaml, whatever) XContentType type = XContentType.values()[between(0, XContentType.values().length - 1)]; XContentBuilder source = XContentFactory.contentBuilder(type); source.startObject().field("text", "findme").endObject(); indexRandom(true, client().prepareIndex("test", "test", "findme").setSource(source)); SuggestResponse response = client().prepareSuggest("test").addSuggestion( - SuggestBuilders.completionSuggestion("test").field("suggest").text("findme")).get(); + SuggestBuilders.oldCompletionSuggestion("test").field("suggest").text("findme")).get(); assertSuggestion(response.getSuggest(), 0, 0, "test", "findme"); CompletionSuggestion.Entry.Option option = (CompletionSuggestion.Entry.Option)response.getSuggest().getSuggestion("test").getEntries().get(0).getOptions().get(0); // And it comes back in exactly that way. diff --git a/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java index 717823d9ffc67..866325f7dbfe6 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/completion/CompletionFieldMapperTests.java @@ -18,21 +18,38 @@ */ package org.elasticsearch.index.mapper.completion; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.suggest.xdocument.*; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; import org.junit.Test; import java.io.IOException; +import java.util.Arrays; import java.util.Map; +import static org.elasticsearch.Version.*; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; +import static org.elasticsearch.test.VersionUtils.randomVersionBetween; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.*; public class CompletionFieldMapperTests extends ESSingleNodeTestCase { @@ -49,22 +66,68 @@ public void testDefaultConfiguration() throws IOException { FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); - CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; - assertThat(completionFieldMapper.isStoringPayloads(), is(false)); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + + NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(true)); + + NamedAnalyzer searchAnalyzer = completionFieldType.searchAnalyzer(); + assertThat(searchAnalyzer.name(), equalTo("simple")); + assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(true)); } @Test - public void testThatSerializationIncludesAllElements() throws Exception { + public void testCompletionAnalyzerSettings() throws Exception { String mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("completion") .field("type", "completion") .field("analyzer", "simple") .field("search_analyzer", "standard") - .field("payloads", true) .field("preserve_separators", false) .field("preserve_position_increments", true) - .field("max_input_length", 14) + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); + + MappedFieldType completionFieldType = fieldMapper.fieldType(); + + NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(false)); + + NamedAnalyzer searchAnalyzer = completionFieldType.searchAnalyzer(); + assertThat(searchAnalyzer.name(), equalTo("standard")); + assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(false)); + + } + @Test + public void testTypeParsing() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .field("analyzer", "simple") + .field("search_analyzer", "standard") + .field("preserve_separators", false) + .field("preserve_position_increments", true) + .field("max_input_length", 14) .endObject().endObject() .endObject().endObject().string(); @@ -75,46 +138,347 @@ public void testThatSerializationIncludesAllElements() throws Exception { CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; XContentBuilder builder = jsonBuilder().startObject(); - completionFieldMapper.toXContent(builder, null).endObject(); + completionFieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); builder.close(); - Map serializedMap; - try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) { - serializedMap = parser.map(); - } + Map serializedMap = JsonXContent.jsonXContent.createParser(builder.bytes()).map(); Map configMap = (Map) serializedMap.get("completion"); assertThat(configMap.get("analyzer").toString(), is("simple")); assertThat(configMap.get("search_analyzer").toString(), is("standard")); - assertThat(Boolean.valueOf(configMap.get("payloads").toString()), is(true)); assertThat(Boolean.valueOf(configMap.get("preserve_separators").toString()), is(false)); assertThat(Boolean.valueOf(configMap.get("preserve_position_increments").toString()), is(true)); assertThat(Integer.valueOf(configMap.get("max_input_length").toString()), is(14)); } @Test - public void testThatSerializationCombinesToOneAnalyzerFieldIfBothAreEqual() throws Exception { + public void testParsingMinimal() throws Exception { String mapping = jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("completion") .field("type", "completion") - .field("analyzer", "simple") - .field("search_analyzer", "simple") .endObject().endObject() .endObject().endObject().string(); DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", "suggestion") + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 1); + } + @Test + public void testBackCompatiblity() throws Exception { + // creating completion field for pre 2.0 indices, should create old completion fields + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + for (Version version : Arrays.asList(V_1_7_0, V_1_1_0, randomVersionBetween(random(), V_1_1_0, V_1_7_0))) { + DocumentMapper defaultMapper = createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version.id).build()) + .mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + assertTrue(fieldMapper instanceof OldCompletionFieldMapper); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", "suggestion") + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertThat(fields.length, equalTo(1)); + assertFalse(fields[0] instanceof SuggestField); + assertAcked(client().admin().indices().prepareDelete("test").execute().get()); + } + // for 2.0 indices and onwards, should create new completion fields + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); - assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); + assertTrue(fieldMapper instanceof CompletionFieldMapper); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", "suggestion") + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertThat(fields.length, equalTo(1)); + assertTrue(fields[0] instanceof SuggestField); + assertAcked(client().admin().indices().prepareDelete("test").execute().get()); + } - CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; - XContentBuilder builder = jsonBuilder().startObject(); - completionFieldMapper.toXContent(builder, null).endObject(); - builder.close(); - Map serializedMap; - try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) { - serializedMap = parser.map(); + @Test + public void testParsingMultiValued() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .array("completion", "suggestion1", "suggestion2") + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 2); + } + + @Test + public void testParsingWithWeight() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("completion") + .field("input", "suggestion") + .field("weight", 2) + .endObject() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 1); + } + + @Test + public void testParsingMultiValueWithWeight() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("completion") + .array("input", "suggestion1", "suggestion2", "suggestion3") + .field("weight", 2) + .endObject() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 3); + } + + @Test + public void testParsingFull() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion1") + .field("weight", 3) + .endObject() + .startObject() + .field("input", "suggestion2") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion3") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 3); + } + + @Test + public void testParsingMixed() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .array("input", "suggestion1", "suggestion2") + .field("weight", 3) + .endObject() + .startObject() + .field("input", "suggestion3") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion4", "suggestion5", "suggestion6") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertSuggestFields(fields, 6); + } + + @Test + public void testNonContextEnabledParsingWithContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("field1") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .startObject("field1") + .field("input", "suggestion1") + .startObject("contexts") + .field("ctx", "ctx2") + .endObject() + .field("weight", 3) + .endObject() + .endObject() + .bytes()); + fail("Supplying contexts to a non context-enabled field should error"); + } catch (MapperParsingException e) { + assertThat(e.getRootCause().getMessage(), containsString("field1")); } - Map configMap = (Map) serializedMap.get("completion"); - assertThat(configMap.get("analyzer").toString(), is("simple")); } + + @Test + public void testFieldValueValidation() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + CharsRefBuilder charsRefBuilder = new CharsRefBuilder(); + charsRefBuilder.append("sugg"); + charsRefBuilder.setCharAt(2, '\u001F'); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", charsRefBuilder.get().toString()) + .endObject() + .bytes()); + fail("No error indexing value with reserved character [0x1F]"); + } catch (MapperParsingException e) { + Throwable cause = e.unwrapCause().getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("[0x1f]")); + } + + charsRefBuilder.setCharAt(2, '\u0000'); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", charsRefBuilder.get().toString()) + .endObject() + .bytes()); + fail("No error indexing value with reserved character [0x0]"); + } catch (MapperParsingException e) { + Throwable cause = e.unwrapCause().getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("[0x0]")); + } + + charsRefBuilder.setCharAt(2, '\u001E'); + try { + defaultMapper.parse("test", "type1", "1", XContentFactory.jsonBuilder() + .startObject() + .field("completion", charsRefBuilder.get().toString()) + .endObject() + .bytes()); + fail("No error indexing value with reserved character [0x1E]"); + } catch (MapperParsingException e) { + Throwable cause = e.unwrapCause().getCause(); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("[0x1e]")); + } + } + + @Test + public void testPrefixQueryType() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; + Query prefixQuery = completionFieldMapper.fieldType().prefixQuery(new BytesRef("co")); + assertThat(prefixQuery, instanceOf(PrefixCompletionQuery.class)); + } + + @Test + public void testFuzzyQueryType() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; + Query prefixQuery = completionFieldMapper.fieldType().fuzzyQuery("co", + Fuzziness.fromEdits(FuzzyCompletionQuery.DEFAULT_MAX_EDITS), FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX, + FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH, Operations.DEFAULT_MAX_DETERMINIZED_STATES, + FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS, FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE); + assertThat(prefixQuery, instanceOf(FuzzyCompletionQuery.class)); + } + + @Test + public void testRegexQueryType() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper; + Query prefixQuery = completionFieldMapper.fieldType() + .regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + assertThat(prefixQuery, instanceOf(RegexCompletionQuery.class)); + } + + private static void assertSuggestFields(IndexableField[] fields, int expected) { + int actualFieldCount = 0; + for (IndexableField field : fields) { + if (field instanceof SuggestField) { + actualFieldCount++; + } + } + assertThat(actualFieldCount, equalTo(expected)); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/completion/OldCompletionFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/completion/OldCompletionFieldMapperTests.java new file mode 100644 index 0000000000000..49e766f8be3bd --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/completion/OldCompletionFieldMapperTests.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper.completion; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.VersionUtils; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class OldCompletionFieldMapperTests extends ESSingleNodeTestCase { + private final Version PRE2X_VERSION = VersionUtils.randomVersionBetween(getRandom(), Version.V_1_0_0, Version.V_1_7_0); + + @Test + public void testDefaultConfiguration() throws IOException { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id).build()) + .mapperService().documentMapperParser().parse(mapping); + + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + assertThat(fieldMapper, instanceOf(OldCompletionFieldMapper.class)); + + OldCompletionFieldMapper completionFieldMapper = (OldCompletionFieldMapper) fieldMapper; + assertThat(completionFieldMapper.isStoringPayloads(), is(false)); + } + + @Test + public void testThatSerializationIncludesAllElements() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .field("analyzer", "simple") + .field("search_analyzer", "standard") + .field("payloads", true) + .field("preserve_separators", false) + .field("preserve_position_increments", true) + .field("max_input_length", 14) + + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id).build()) + .mapperService().documentMapperParser().parse(mapping); + + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + assertThat(fieldMapper, instanceOf(OldCompletionFieldMapper.class)); + + OldCompletionFieldMapper completionFieldMapper = (OldCompletionFieldMapper) fieldMapper; + XContentBuilder builder = jsonBuilder().startObject(); + completionFieldMapper.toXContent(builder, null).endObject(); + builder.close(); + Map serializedMap; + try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) { + serializedMap = parser.map(); + } + Map configMap = (Map) serializedMap.get("completion"); + assertThat(configMap.get("analyzer").toString(), is("simple")); + assertThat(configMap.get("search_analyzer").toString(), is("standard")); + assertThat(Boolean.valueOf(configMap.get("payloads").toString()), is(true)); + assertThat(Boolean.valueOf(configMap.get("preserve_separators").toString()), is(false)); + assertThat(Boolean.valueOf(configMap.get("preserve_position_increments").toString()), is(true)); + assertThat(Integer.valueOf(configMap.get("max_input_length").toString()), is(14)); + } + + @Test + public void testThatSerializationCombinesToOneAnalyzerFieldIfBothAreEqual() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .field("analyzer", "simple") + .field("search_analyzer", "simple") + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id).build()) + .mapperService().documentMapperParser().parse(mapping); + + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + assertThat(fieldMapper, instanceOf(OldCompletionFieldMapper.class)); + + OldCompletionFieldMapper completionFieldMapper = (OldCompletionFieldMapper) fieldMapper; + XContentBuilder builder = jsonBuilder().startObject(); + completionFieldMapper.toXContent(builder, null).endObject(); + builder.close(); + Map serializedMap; + try (XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes())) { + serializedMap = parser.map(); + } + Map configMap = (Map) serializedMap.get("completion"); + assertThat(configMap.get("analyzer").toString(), is("simple")); + } + +} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/OldCompletionFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/OldCompletionFieldTypeTests.java new file mode 100644 index 0000000000000..8dcbf8e8db7b3 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/OldCompletionFieldTypeTests.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper.core; + +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; + +public class OldCompletionFieldTypeTests extends FieldTypeTestCase { + @Override + protected MappedFieldType createDefaultFieldType() { + return new OldCompletionFieldMapper.CompletionFieldType(); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java index f3636bf4c4713..79bbded1df59a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java @@ -19,15 +19,19 @@ package org.elasticsearch.index.mapper.multifield; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.junit.Test; import java.io.IOException; @@ -172,9 +176,11 @@ public void testTokenCountMultiField() throws Exception { } @Test - public void testCompletionMultiField() throws Exception { + public void testPre2xCompletionMultiField() throws Exception { + final Version PRE2X_VERSION = VersionUtils.randomVersionBetween(getRandom(), Version.V_1_0_0, Version.V_1_7_0); assertAcked( client().admin().indices().prepareCreate("my-index") + .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) .addMapping("my-type", createMappingSource("completion")) ); @@ -197,6 +203,32 @@ public void testCompletionMultiField() throws Exception { assertThat(countResponse.getCount(), equalTo(1l)); } + @Test + public void testCompletionMultiField() throws Exception { + assertAcked( + client().admin().indices().prepareCreate("my-index") + .addMapping("my-type", createMappingSource("completion")) + ); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get(); + MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type"); + assertThat(mappingMetaData, not(nullValue())); + Map mappingSource = mappingMetaData.sourceAsMap(); + Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); + assertThat(aField.size(), equalTo(6)); + assertThat(aField.get("type").toString(), equalTo("completion")); + assertThat(aField.get("fields"), notNullValue()); + + Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource)); + assertThat(bField.size(), equalTo(2)); + assertThat(bField.get("type").toString(), equalTo("string")); + assertThat(bField.get("index").toString(), equalTo("not_analyzed")); + + client().prepareIndex("my-index", "my-type", "1").setSource("a", "complete me").setRefresh(true).get(); + CountResponse countResponse = client().prepareCount("my-index").setQuery(matchQuery("a.b", "complete me")).get(); + assertThat(countResponse.getCount(), equalTo(1l)); + } + @Test public void testIpMultiField() throws Exception { assertAcked( diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 8aeb6f65b9b55..3ffad3568f7d9 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -51,6 +51,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.search.suggest.SuggestBuilders; import org.junit.Test; import java.io.IOException; @@ -748,8 +749,18 @@ public void testCompletionFieldsParam() throws Exception { client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet(); refresh(); + // should not load FST in memory yet IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); IndicesStatsResponse stats = builder.execute().actionGet(); + assertThat(stats.getTotal().completion.getSizeInBytes(), equalTo(0l)); + assertThat(stats.getTotal().completion.getFields(), is(nullValue())); + + // load FST in memory by searching + client().prepareSuggest().addSuggestion(SuggestBuilders.completionSuggestion("suggest").field("bar.completion").prefix("ba")).get(); + client().prepareSuggest().addSuggestion(SuggestBuilders.completionSuggestion("suggest").field("baz.completion").prefix("ba")).get(); + + builder = client().admin().indices().prepareStats(); + stats = builder.execute().actionGet(); assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0l)); assertThat(stats.getTotal().completion.getFields(), is(nullValue())); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index 672dc5e081fb6..036ac86549c22 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -21,8 +21,10 @@ import com.carrotsearch.hppc.ObjectLongHashMap; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.google.common.collect.Lists; - +import org.apache.lucene.analysis.TokenStreamToAutomaton; +import org.apache.lucene.search.suggest.xdocument.ContextSuggestField; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; @@ -33,12 +35,11 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; import org.elasticsearch.percolator.PercolatorService; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -46,24 +47,20 @@ import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder.FuzzyOptionsBuilder; +import org.elasticsearch.search.suggest.completion.context.*; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Test; import java.io.IOException; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.ExecutionException; +import java.util.*; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.*; @SuppressCodecs("*") // requires custom completion format @@ -75,27 +72,120 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { private final CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); @Test - public void testSimple() throws Exception { - createIndexAndMapping(completionMappingBuilder); - String[][] input = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, - {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, - {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, - {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies - for (int i = 0; i < input.length; i++) { - client().prepareIndex(INDEX, TYPE, "" + i) + public void testPrefix() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .endObject() - .endObject() - ) - .execute().actionGet(); + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i) + .endObject() + .endObject() + )); } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); + } - refresh(); + @Test + public void testRegex() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugg" + i + "estion") + .field("weight", i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("sugg.*es"); + assertSuggestions("foo", prefix, "sugg10estion", "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion"); + } - assertSuggestionsNotInOrder("f", "Foo Fighters", "Firestarter", "Foo Fighters Generator", "Foo Fighters Learn to Fly"); - assertSuggestionsNotInOrder("t", "The Prodigy", "Turbonegro", "Turbonegro Get it on", "The Prodigy Firestarter"); + @Test + public void testFuzzy() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugxgestion"+i) + .field("weight", i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE); + assertSuggestions("foo", prefix, "sugxgestion10", "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6"); + } + + @Test + public void testMixedCompletion() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + String otherIndex = INDEX + "_1"; + assertAcked(client().admin().indices().prepareCreate(otherIndex) + .setSettings(Settings.settingsBuilder().put(indexSettings()).put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_7_0.id)) + .addMapping(TYPE, jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .field("analyzer", mapping.indexAnalyzer) + .field("search_analyzer", mapping.searchAnalyzer) + .field("preserve_separators", mapping.preserveSeparators) + .field("preserve_position_increments", mapping.preservePositionIncrements) + .endObject() + .endObject().endObject() + .endObject()) + .get()); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(otherIndex, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i) + .endObject() + .endObject() + )); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i) + .endObject() + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).text("sugg"); + try { + client().prepareSuggest(INDEX, otherIndex).addSuggestion(prefix).execute().actionGet(); + fail("querying on mixed completion suggester should throw an error"); + } catch (IllegalArgumentException e) { + // expected + } } @Test @@ -128,19 +218,6 @@ public void testSuggestFieldWithPercolateApi() throws Exception { assertThat(response.getCount(), equalTo(1l)); } - @Test - public void testBasicPrefixSuggestion() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - for (int i = 0; i < 2; i++) { - createData(i == 0); - assertSuggestions("f", "Firestarter - The Prodigy", "Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters"); - assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro"); - assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro"); - assertSuggestions("t", "The Prodigy", "Firestarter - The Prodigy", "Get it on - Turbonegro", "Turbonegro"); - } - } - @Test public void testThatWeightsAreWorking() throws Exception { createIndexAndMapping(completionMappingBuilder); @@ -245,121 +322,15 @@ public void testThatInputCanBeAStringInsteadOfAnArray() throws Exception { createIndexAndMapping(completionMappingBuilder); client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .field("input", "Foo Fighters") - .field("output", "Boo Fighters") - .endObject().endObject() - ).get(); - - refresh(); - - assertSuggestions("f", "Boo Fighters"); - } - - @Test - public void testThatPayloadsAreArbitraryJsonObjects() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .startObject("payload").field("foo", "bar").startArray("test").value("spam").value("eggs").endArray().endObject() - .endObject().endObject() - ).get(); - - refresh(); - - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) - ).execute().actionGet(); - - assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); - assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); - CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; - assertThat(prefixOption.getPayload(), is(notNullValue())); - - // parse JSON - Map jsonMap = prefixOption.getPayloadAsMap(); - assertThat(jsonMap.size(), is(2)); - assertThat(jsonMap.get("foo").toString(), is("bar")); - assertThat(jsonMap.get("test"), is(instanceOf(List.class))); - List listValues = (List) jsonMap.get("test"); - assertThat(listValues, hasItems("spam", "eggs")); - } - - @Test - public void testPayloadAsNumeric() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .field("payload", 1) - .endObject().endObject() - ).get(); - - refresh(); - - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) - ).execute().actionGet(); - - assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); - assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); - CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; - assertThat(prefixOption.getPayload(), is(notNullValue())); - - assertThat(prefixOption.getPayloadAsLong(), equalTo(1l)); - } - - @Test - public void testPayloadAsString() throws Exception { - completionMappingBuilder.payloads(true); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .field("payload", "test") - .endObject().endObject() + .startObject().startObject(FIELD) + .field("input", "Foo Fighters") + .endObject().endObject() ).get(); refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) - ).execute().actionGet(); - - assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); - assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); - CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; - assertThat(prefixOption.getPayload(), is(notNullValue())); - - assertThat(prefixOption.getPayloadAsString(), equalTo("test")); - } - - @Test(expected = MapperException.class) - public void testThatExceptionIsThrownWhenPayloadsAreDisabledButInIndexRequest() throws Exception { - completionMappingBuilder.payloads(false); - createIndexAndMapping(completionMappingBuilder); - - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value("Foo Fighters").endArray() - .field("output", "Boo Fighters") - .startArray("payload").value("spam").value("eggs").endArray() - .endObject().endObject() - ).get(); + assertSuggestions("f", "Foo Fighters"); } - @Test public void testDisabledPreserveSeparators() throws Exception { completionMappingBuilder.preserveSeparators(false); @@ -413,14 +384,13 @@ public void testThatMultipleInputsAreSupported() throws Exception { client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() .startObject().startObject(FIELD) .startArray("input").value("Foo Fighters").value("Fu Fighters").endArray() - .field("output", "The incredible Foo Fighters") .endObject().endObject() ).get(); refresh(); - assertSuggestions("foo", "The incredible Foo Fighters"); - assertSuggestions("fu", "The incredible Foo Fighters"); + assertSuggestions("foo", "Foo Fighters"); + assertSuggestions("fu", "Fu Fighters"); } @Test @@ -579,12 +549,12 @@ public void testThatFuzzySuggesterWorks() throws Exception { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirv").size(10) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirv").size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirw").size(10) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirw", Fuzziness.ONE).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } @@ -603,13 +573,13 @@ public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { // edit distance 1 SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Norw").size(10) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Norw", Fuzziness.ONE).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); // edit distance 2 suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Norw").size(10).setFuzziness(Fuzziness.TWO) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Norw", Fuzziness.TWO).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } @@ -627,12 +597,12 @@ public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(false).setFuzziness(Fuzziness.ONE) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriv", new FuzzyOptionsBuilder().setTranspositions(false)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(true).setFuzziness(Fuzziness.ONE) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriv", Fuzziness.ONE).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } @@ -650,12 +620,12 @@ public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nriva").size(10).setFuzzyMinLength(6) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nriva", new FuzzyOptionsBuilder().setFuzzyMinLength(6)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nrivan").size(10).setFuzzyMinLength(6) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nrivan", new FuzzyOptionsBuilder().setFuzzyMinLength(6)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } @@ -673,12 +643,12 @@ public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { refresh(); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirw").size(10).setFuzzyPrefixLength(4) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirw", new FuzzyOptionsBuilder().setFuzzyPrefixLength(4)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("Nirvo").size(10).setFuzzyPrefixLength(4) + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("Nirvo", new FuzzyOptionsBuilder().setFuzzyPrefixLength(4)).size(10) ).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "Nirvana"); } @@ -696,19 +666,19 @@ public void testThatFuzzySuggesterIsUnicodeAware() throws Exception { refresh(); // suggestion with a character, which needs unicode awareness - CompletionSuggestionFuzzyBuilder completionSuggestionBuilder = - SuggestBuilders.fuzzyCompletionSuggestion("foo").field(FIELD).text("öööи").size(10).setUnicodeAware(true); + org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder completionSuggestionBuilder = + SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(true)).size(10); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "ööööö"); // removing unicode awareness leads to no result - completionSuggestionBuilder.setUnicodeAware(false); + completionSuggestionBuilder = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(false)).size(10); suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo"); // increasing edit distance instead of unicode awareness works again, as this is only a single character - completionSuggestionBuilder.setFuzziness(Fuzziness.TWO); + completionSuggestionBuilder = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("öööи", new FuzzyOptionsBuilder().setUnicodeAware(false).setFuzziness(Fuzziness.TWO)).size(10); suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); assertSuggestions(suggestResponse, false, "foo", "ööööö"); } @@ -721,7 +691,7 @@ public void testThatStatsAreWorking() throws Exception { PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() .startObject(TYPE).startObject("properties") - .startObject(FIELD.toString()) + .startObject(FIELD) .field("type", "completion").field("analyzer", "simple") .endObject() .startObject(otherField) @@ -732,8 +702,15 @@ public void testThatStatsAreWorking() throws Exception { assertThat(putMappingResponse.isAcknowledged(), is(true)); // Index two entities - client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()).get(); - client().prepareIndex(INDEX, TYPE, "2").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()).get(); + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()).get(); + client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()).get(); + + refresh(); + ensureGreen(); + // load the fst index into ram + client().prepareSuggest(INDEX).addSuggestion(SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("f")).get(); + client().prepareSuggest(INDEX).addSuggestion(SuggestBuilders.completionSuggestion("foo").field(otherField).prefix("f")).get(); + refresh(); // Get all stats IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get(); @@ -828,13 +805,16 @@ public void testThatIndexingInvalidFieldsInCompletionFieldResultsInException() t } - public void assertSuggestions(String suggestion, String... suggestions) { - String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( - SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestion).size(10) + public void assertSuggestions(String suggestionName, SuggestBuilder.SuggestionBuilder suggestBuilder, String... suggestions) { + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestBuilder ).execute().actionGet(); - assertSuggestions(suggestResponse, suggestionName, suggestions); + + } + public void assertSuggestions(String suggestion, String... suggestions) { + String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggestion).size(10); + assertSuggestions(suggestionName, suggestionBuilder, suggestions); } public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) { @@ -846,11 +826,11 @@ public void assertSuggestionsNotInOrder(String suggestString, String... suggesti assertSuggestions(suggestResponse, false, suggestionName, suggestions); } - private void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { + static void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { assertSuggestions(suggestResponse, true, name, suggestions); } - private void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { + private static void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { assertAllSuccessful(suggestResponse); List suggestionNames = Lists.newArrayList(); @@ -880,7 +860,7 @@ private void assertSuggestions(SuggestResponse suggestResponse, boolean suggesti } } - private List getNames(Suggest.Suggestion.Entry suggestEntry) { + private static List getNames(Suggest.Suggestion.Entry suggestEntry) { List names = Lists.newArrayList(); for (Suggest.Suggestion.Entry.Option entry : suggestEntry.getOptions()) { names.add(entry.getText().string()); @@ -889,20 +869,44 @@ private List getNames(Suggest.Suggestion.Entry> contextMapping : completionMappingBuilder.contextMappings.entrySet()) { + mapping = mapping.startObject() + .field("name", contextMapping.getValue().name()) + .field("type", contextMapping.getValue().type().name()); + switch (contextMapping.getValue().type()) { + case CATEGORY: + mapping = mapping.field("path", ((CategoryContextMapping) contextMapping.getValue()).getFieldName()); + break; + case GEO: + mapping = mapping + .field("path", ((GeoContextMapping) contextMapping.getValue()).getFieldName()) + .field("precision", ((GeoContextMapping) contextMapping.getValue()).getPrecision()); + break; + } + + mapping = mapping.endObject(); + } + + mapping = mapping.endArray(); + } + mapping = mapping.endObject() + .endObject().endObject() + .endObject(); + assertAcked(client().admin().indices().prepareCreate(INDEX) .setSettings(Settings.settingsBuilder().put(indexSettings()).put(settings)) - .addMapping(TYPE, jsonBuilder().startObject() - .startObject(TYPE).startObject("properties") - .startObject(FIELD) - .field("type", "completion") - .field("analyzer", completionMappingBuilder.indexAnalyzer) - .field("search_analyzer", completionMappingBuilder.searchAnalyzer) - .field("payloads", completionMappingBuilder.payloads) - .field("preserve_separators", completionMappingBuilder.preserveSeparators) - .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements) - .endObject() - .endObject().endObject() - .endObject()) + .addMapping(TYPE, mapping) .get()); ensureYellow(); } @@ -911,47 +915,6 @@ private void createIndexAndMapping(CompletionMappingBuilder completionMappingBui createIndexAndMappingAndSettings(Settings.EMPTY, completionMappingBuilder); } - private void createData(boolean optimize) throws IOException, InterruptedException, ExecutionException { - String[][] input = {{"Foo Fighters"}, {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; - String[] surface = {"Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters", "The Prodigy", "Firestarter - The Prodigy", "Turbonegro", "Get it on - Turbonegro"}; - int[] weight = {10, 9, 8, 12, 11, 6, 7}; - IndexRequestBuilder[] builders = new IndexRequestBuilder[input.length]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex(INDEX, TYPE, "" + i) - .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .field("output", surface[i]) - .startObject("payload").field("id", i).endObject() - .field("weight", 1) // WE FORCEFULLY INDEX A BOGUS WEIGHT - .endObject() - .endObject() - ); - } - indexRandom(false, builders); - - for (int i = 0; i < builders.length; i++) { // add them again to make sure we deduplicate on the surface form - builders[i] = client().prepareIndex(INDEX, TYPE, "n" + i) - .setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(input[i]).endArray() - .field("output", surface[i]) - .startObject("payload").field("id", i).endObject() - .field("weight", weight[i]) - .endObject() - .endObject() - ); - } - indexRandom(false, builders); - - client().admin().indices().prepareRefresh(INDEX).execute().actionGet(); - if (optimize) { - // make sure merging works just fine - client().admin().indices().prepareFlush(INDEX).execute().actionGet(); - client().admin().indices().prepareOptimize(INDEX).setMaxNumSegments(randomIntBetween(1, 5)).get(); - } - } - @Test // see #3555 public void testPrunedSegments() throws IOException { createIndexAndMappingAndSettings(settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build(), completionMappingBuilder); @@ -989,44 +952,6 @@ public void testPrunedSegments() throws IOException { } } - @Test - public void testMaxFieldLength() throws IOException { - client().admin().indices().prepareCreate(INDEX).get(); - ensureGreen(); - int iters = scaledRandomIntBetween(10, 20); - for (int i = 0; i < iters; i++) { - int maxInputLen = between(3, 50); - String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(maxInputLen + 1, maxInputLen + scaledRandomIntBetween(2, 50)), (char) 0x01); - assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() - .startObject(TYPE).startObject("properties") - .startObject(FIELD) - .field("type", "completion") - .field("max_input_length", maxInputLen) - // upgrade mapping each time - .field("analyzer", "keyword") - .endObject() - .endObject().endObject() - .endObject())); - client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject().startObject(FIELD) - .startArray("input").value(str).endArray() - .field("output", "foobar") - .endObject().endObject() - ).setRefresh(true).get(); - // need to flush and refresh, because we keep changing the same document - // we have to make sure that segments without any live documents are deleted - flushAndRefresh(); - int prefixLen = CompletionFieldMapper.correctSubStringLen(str, between(1, maxInputLen - 1)); - assertSuggestions(str.substring(0, prefixLen), "foobar"); - if (maxInputLen + 1 < str.length()) { - int offset = Character.isHighSurrogate(str.charAt(maxInputLen - 1)) ? 2 : 1; - int correctSubStringLen = CompletionFieldMapper.correctSubStringLen(str, maxInputLen + offset); - String shortenedSuggestion = str.substring(0, correctSubStringLen); - assertSuggestions(shortenedSuggestion); - } - } - } - @Test // see #3596 public void testVeryLongInput() throws IOException { @@ -1043,7 +968,6 @@ public void testVeryLongInput() throws IOException { client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() .startObject().startObject(FIELD) .startArray("input").value(longString).endArray() - .field("output", "foobar") .endObject().endObject() ).setRefresh(true).get(); @@ -1082,9 +1006,9 @@ public void testIssue5930() throws IOException { ensureYellow(); String string = "foo bar"; client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() - .startObject() - .field(FIELD, string) - .endObject() + .startObject() + .field(FIELD, string) + .endObject() ).setRefresh(true).get(); try { @@ -1116,7 +1040,7 @@ public void testIndexingUnrelatedNullValue() throws Exception { ensureGreen(); client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "strings make me happy", FIELD + "_1", "nulls make me sad") - .setRefresh(true).get(); + .setRefresh(true).get(); try { client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad") @@ -1129,22 +1053,34 @@ public void testIndexingUnrelatedNullValue() throws Exception { } + public static boolean isReservedChar(char c) { + switch (c) { + case '\u001F': + case TokenStreamToAutomaton.HOLE: + case 0x0: + case ContextSuggestField.CONTEXT_SEPARATOR: + return true; + default: + return false; + } + } + private static String replaceReservedChars(String input, char replacement) { char[] charArray = input.toCharArray(); for (int i = 0; i < charArray.length; i++) { - if (CompletionFieldMapper.isReservedChar(charArray[i])) { + if (isReservedChar(charArray[i])) { charArray[i] = replacement; } } return new String(charArray); } - private static class CompletionMappingBuilder { - private String searchAnalyzer = "simple"; - private String indexAnalyzer = "simple"; - private Boolean payloads = getRandom().nextBoolean(); - private Boolean preserveSeparators = getRandom().nextBoolean(); - private Boolean preservePositionIncrements = getRandom().nextBoolean(); + static class CompletionMappingBuilder { + String searchAnalyzer = "simple"; + String indexAnalyzer = "simple"; + Boolean preserveSeparators = getRandom().nextBoolean(); + Boolean preservePositionIncrements = getRandom().nextBoolean(); + LinkedHashMap> contextMappings = null; public CompletionMappingBuilder searchAnalyzer(String searchAnalyzer) { this.searchAnalyzer = searchAnalyzer; @@ -1154,10 +1090,6 @@ public CompletionMappingBuilder indexAnalyzer(String indexAnalyzer) { this.indexAnalyzer = indexAnalyzer; return this; } - public CompletionMappingBuilder payloads(Boolean payloads) { - this.payloads = payloads; - return this; - } public CompletionMappingBuilder preserveSeparators(Boolean preserveSeparators) { this.preserveSeparators = preserveSeparators; return this; @@ -1166,5 +1098,10 @@ public CompletionMappingBuilder preservePositionIncrements(Boolean preservePosit this.preservePositionIncrements = preservePositionIncrements; return this; } + + public CompletionMappingBuilder context(LinkedHashMap> contextMappings) { + this.contextMappings = contextMappings; + return this; + } } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTest.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTest.java index 53e17966968e0..156d105a6b0a8 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTest.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionTokenStreamTest.java @@ -31,8 +31,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.IntsRef; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream; -import org.elasticsearch.search.suggest.completion.CompletionTokenStream.ByteTermAttribute; +import org.elasticsearch.search.suggest.completion.old.CompletionTokenStream; +import org.elasticsearch.search.suggest.completion.old.CompletionTokenStream.ByteTermAttribute; import org.elasticsearch.test.ESTokenStreamTestCase; import org.junit.Test; diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java new file mode 100644 index 0000000000000..5826344f4a91a --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -0,0 +1,596 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.common.geo.GeoHashUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.suggest.CompletionSuggestSearchIT.CompletionMappingBuilder; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; +import org.elasticsearch.search.suggest.completion.context.*; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +@SuppressCodecs("*") // requires custom completion format +public class ContextCompletionSuggestSearchIT extends ESIntegTestCase { + + /* TODO: currently to get n completions with contexts, we have to request size of + * n * size(contexts); internally a suggestion + a context value is considered + * as one hit, instead of a suggestion + all its context values + */ + + private final String INDEX = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final String TYPE = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final String FIELD = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + + @Test + public void testContextPrefix() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + boolean addAnotherContext = randomBoolean(); + if (addAnotherContext) { + map.put("type", ContextBuilder.category("type").field("type").build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2); + if (addAnotherContext) { + source.field("type", "type" + i % 3); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(5 * (addAnotherContext ? 2 : 1)); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + @Test + public void testContextRegex() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + boolean addAnotherContext = randomBoolean(); + if (addAnotherContext) { + map.put("type", ContextBuilder.category("type").field("type").build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugg" + i + "estion") + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2); + if (addAnotherContext) { + source.field("type", "type" + i % 3); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).regex("sugg.*es").size(5 * (addAnotherContext ? 2 : 1)); + assertSuggestions("foo", prefix, "sugg9estion", "sugg8estion", "sugg7estion", "sugg6estion", "sugg5estion"); + } + + @Test + public void testContextFuzzy() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + boolean addAnotherContext = randomBoolean(); + if (addAnotherContext) { + map.put("type", ContextBuilder.category("type").field("type").build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "sugxgestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2); + if (addAnotherContext) { + source.field("type", "type" + i % 3); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg", Fuzziness.ONE).size(5 * (addAnotherContext ? 2 : 1)); + assertSuggestions("foo", prefix, "sugxgestion9", "sugxgestion8", "sugxgestion7", "sugxgestion6", "sugxgestion5"); + } + + @Test + public void testSingleContextFiltering() throws Exception { + CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build(); + LinkedHashMap> map = new LinkedHashMap>(Collections.singletonMap("cat", contextMapping)); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .categoryContexts("cat", + new CategoryQueryContext("cat0")); + + assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0"); + } + + @Test + public void testSingleContextBoosting() throws Exception { + CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build(); + LinkedHashMap> map = new LinkedHashMap>(Collections.singletonMap("cat", contextMapping)); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .categoryContexts("cat", + new CategoryQueryContext("cat0", 3), + new CategoryQueryContext("cat1")); + + assertSuggestions("foo", prefix, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2"); + } + + @Test + public void testSingleContextMultipleContexts() throws Exception { + CategoryContextMapping contextMapping = ContextBuilder.category("cat").field("cat").build(); + LinkedHashMap> map = new LinkedHashMap>(Collections.singletonMap("cat", contextMapping)); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List contexts = Arrays.asList("type1", "type2", "type3", "type4"); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", contexts) + .endObject() + )); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(5 * contexts.size()); + + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + @Override + protected int numberOfShards() { + return 1; + } + @Test + public void testMultiContextFiltering() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + map.put("type", ContextBuilder.category("type").field("type").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .field("type", "type" + i % 4) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + // filter only on context cat + CompletionSuggestionBuilder catFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + catFilterSuggest.categoryContexts("cat", new CategoryQueryContext("cat0")); + assertSuggestions("foo", catFilterSuggest, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0"); + + // filter only on context type + CompletionSuggestionBuilder typeFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + typeFilterSuggest.categoryContexts("type", new CategoryQueryContext("type2"), new CategoryQueryContext("type1")); + assertSuggestions("foo", typeFilterSuggest, "suggestion9", "suggestion6", "suggestion5", "suggestion2", "suggestion1"); + + // filter on both contexts + CompletionSuggestionBuilder multiContextFilterSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + // query context order should never matter + if (randomBoolean()) { + multiContextFilterSuggest.categoryContexts("type", new CategoryQueryContext("type2"), new CategoryQueryContext("type1")); + multiContextFilterSuggest.categoryContexts("cat", new CategoryQueryContext("cat0")); + } else { + multiContextFilterSuggest.categoryContexts("cat", new CategoryQueryContext("cat0")); + multiContextFilterSuggest.categoryContexts("type", new CategoryQueryContext("type2"), new CategoryQueryContext("type1")); + } + assertSuggestions("foo", multiContextFilterSuggest, "suggestion9", "suggestion8", "suggestion6", "suggestion5"); + } + + @Test + @AwaitsFix(bugUrl = "multiple context boosting is broken, as a suggestion, contexts pair is treated as (num(context) entries)") + public void testMultiContextBoosting() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + map.put("type", ContextBuilder.category("type").field("type").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject() + .field("cat", "cat" + i % 2) + .field("type", "type" + i % 4) + .endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + // boost only on context cat + CompletionSuggestionBuilder catBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + catBoostSuggest.categoryContexts("cat", + new CategoryQueryContext("cat0", 3), + new CategoryQueryContext("cat1")); + assertSuggestions("foo", catBoostSuggest, "suggestion8", "suggestion6", "suggestion4", "suggestion9", "suggestion2"); + + // boost only on context type + CompletionSuggestionBuilder typeBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + typeBoostSuggest.categoryContexts("type", + new CategoryQueryContext("type2", 2), + new CategoryQueryContext("type1", 4)); + assertSuggestions("foo", typeBoostSuggest, "suggestion9", "suggestion5", "suggestion6", "suggestion1", "suggestion2"); + + // boost on both contexts + CompletionSuggestionBuilder multiContextBoostSuggest = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + // query context order should never matter + if (randomBoolean()) { + multiContextBoostSuggest.categoryContexts("type", + new CategoryQueryContext("type2", 2), + new CategoryQueryContext("type1", 4)); + multiContextBoostSuggest.categoryContexts("cat", + new CategoryQueryContext("cat0", 3), + new CategoryQueryContext("cat1")); + } else { + multiContextBoostSuggest.categoryContexts("cat", + new CategoryQueryContext("cat0", 3), + new CategoryQueryContext("cat1")); + multiContextBoostSuggest.categoryContexts("type", + new CategoryQueryContext("type2", 2), + new CategoryQueryContext("type1", 4)); + } + assertSuggestions("foo", multiContextBoostSuggest, "suggestion9", "suggestion6", "suggestion5", "suggestion2", "suggestion1"); + } + + @Test + public void testMissingContextValue() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("cat", ContextBuilder.category("cat").field("cat").build()); + map.put("type", ContextBuilder.category("type").field("type").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .endObject(); + if (randomBoolean()) { + source.field("cat", "cat" + i % 2); + } + if (randomBoolean()) { + source.field("type", "type" + i % 4); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(5 * 2); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + @Test + public void testSeveralContexts() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + final int numContexts = randomIntBetween(2, 5); + for (int i = 0; i < numContexts; i++) { + map.put("type"+i, ContextBuilder.category("type"+i).field("type" + i).build()); + } + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = randomIntBetween(10, 200); + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", numDocs - i) + .endObject(); + for (int c = 0; c < numContexts; c++) { + source.field("type"+c, "type" + c +i % 4); + } + source.endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg").size(5 * numContexts); + assertSuggestions("foo", prefix, "suggestion0", "suggestion1", "suggestion2", "suggestion3", "suggestion4"); + } + + @Test + public void testSimpleGeoPrefix() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", GeoHashUtils.encode(1.2, 1.3)) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + @Test + public void testGeoFiltering() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + String[] geoHashes = new String[] {"ezs42e44yx96", "u4pruydqqvj8"}; + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", (i % 2 == 0) ? geoHashes[0] : geoHashes[1]) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + + CompletionSuggestionBuilder geoFilteringPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", new GeoQueryContext(geoHashes[0])); + + assertSuggestions("foo", geoFilteringPrefix, "suggestion8", "suggestion6", "suggestion4", "suggestion2", "suggestion0"); + } + + @Test + public void testGeoBoosting() throws Exception { + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + String[] geoHashes = new String[] {"ezs42e44yx96", "u4pruydqqvj8"}; + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", (i % 2 == 0) ? geoHashes[0] : geoHashes[1]) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + + CompletionSuggestionBuilder geoBoostingPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", new GeoQueryContext(geoHashes[0], 2), new GeoQueryContext(geoHashes[1])); + + assertSuggestions("foo", geoBoostingPrefix, "suggestion8", "suggestion6", "suggestion9", "suggestion4", "suggestion7"); + } + + @Test + public void testGeoNeighbours() throws Exception { + String geohash = "gcpv"; + List neighbours = new ArrayList<>(); + neighbours.add("gcpw"); + neighbours.add("gcpy"); + neighbours.add("u10n"); + neighbours.add("gcpt"); + neighbours.add("u10j"); + neighbours.add("gcps"); + neighbours.add("gcpu"); + neighbours.add("u10h"); + + LinkedHashMap> map = new LinkedHashMap<>(); + map.put("geo", ContextBuilder.geo("geo").precision(4).build()); + final CompletionMappingBuilder mapping = new CompletionMappingBuilder().context(map); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + XContentBuilder source = jsonBuilder() + .startObject() + .startObject(FIELD) + .field("input", "suggestion" + i) + .field("weight", i + 1) + .startObject("contexts") + .field("geo", randomFrom(neighbours)) + .endObject() + .endObject().endObject(); + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(source)); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg"); + assertSuggestions("foo", prefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + + CompletionSuggestionBuilder geoNeighbourPrefix = SuggestBuilders.completionSuggestion("foo").field(FIELD).prefix("sugg") + .geoContexts("geo", new GeoQueryContext(geohash)); + + assertSuggestions("foo", geoNeighbourPrefix, "suggestion9", "suggestion8", "suggestion7", "suggestion6", "suggestion5"); + } + + public void assertSuggestions(String suggestionName, SuggestBuilder.SuggestionBuilder suggestBuilder, String... suggestions) { + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestBuilder + ).execute().actionGet(); + CompletionSuggestSearchIT.assertSuggestions(suggestResponse, suggestionName, suggestions); + } + + private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException { + createIndexAndMappingAndSettings(Settings.EMPTY, completionMappingBuilder); + } + private void createIndexAndMappingAndSettings(Settings settings, CompletionMappingBuilder completionMappingBuilder) throws IOException { + XContentBuilder mapping = jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .field("analyzer", completionMappingBuilder.indexAnalyzer) + .field("search_analyzer", completionMappingBuilder.searchAnalyzer) + .field("preserve_separators", completionMappingBuilder.preserveSeparators) + .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements); + + if (completionMappingBuilder.contextMappings != null) { + mapping = mapping.startArray("contexts"); + for (Map.Entry> contextMapping : completionMappingBuilder.contextMappings.entrySet()) { + mapping = mapping.startObject() + .field("name", contextMapping.getValue().name()) + .field("type", contextMapping.getValue().type().name()); + switch (contextMapping.getValue().type()) { + case CATEGORY: + final String fieldName = ((CategoryContextMapping) contextMapping.getValue()).getFieldName(); + if (fieldName != null) { + mapping = mapping.field("path", fieldName); + } + break; + case GEO: + final String name = ((GeoContextMapping) contextMapping.getValue()).getFieldName(); + mapping = mapping + .field("precision", ((GeoContextMapping) contextMapping.getValue()).getPrecision()); + if (name != null) { + mapping.field("path", name); + } + break; + } + + mapping = mapping.endObject(); + } + + mapping = mapping.endArray(); + } + mapping = mapping.endObject() + .endObject().endObject() + .endObject(); + + assertAcked(client().admin().indices().prepareCreate(INDEX) + .setSettings(Settings.settingsBuilder().put(indexSettings()).put(settings)) + .addMapping(TYPE, mapping) + .get()); + ensureYellow(); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/OldCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/OldCompletionSuggestSearchIT.java new file mode 100644 index 0000000000000..90fa4eaecb2a0 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/OldCompletionSuggestSearchIT.java @@ -0,0 +1,1175 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.suggest; + +import com.carrotsearch.hppc.ObjectLongHashMap; +import com.carrotsearch.randomizedtesting.generators.RandomStrings; +import com.google.common.collect.Lists; + +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; +import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; +import org.elasticsearch.action.admin.indices.segments.ShardSegments; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.percolate.PercolateResponse; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.MapperException; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; +import org.elasticsearch.percolator.PercolatorService; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestion; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestionBuilder; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestionFuzzyBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; +import static org.hamcrest.Matchers.*; + +@SuppressCodecs("*") // requires custom completion format +public class OldCompletionSuggestSearchIT extends ESIntegTestCase { + + private final String INDEX = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final String TYPE = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final String FIELD = RandomStrings.randomAsciiOfLength(getRandom(), 10).toLowerCase(Locale.ROOT); + private final Version PRE2X_VERSION = VersionUtils.randomVersionBetween(getRandom(), Version.V_1_0_0, Version.V_1_7_0); + private final CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); + + @Test + public void testSimple() throws Exception { + createIndexAndMapping(completionMappingBuilder); + String[][] input = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, + {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, + {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, + {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies + for (int i = 0; i < input.length; i++) { + client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(input[i]).endArray() + .endObject() + .endObject() + ) + .execute().actionGet(); + } + + refresh(); + + assertSuggestionsNotInOrder("f", "Foo Fighters", "Firestarter", "Foo Fighters Generator", "Foo Fighters Learn to Fly"); + assertSuggestionsNotInOrder("t", "The Prodigy", "Turbonegro", "Turbonegro Get it on", "The Prodigy Firestarter"); + } + + @Test + public void testSuggestFieldWithPercolateApi() throws Exception { + createIndexAndMapping(completionMappingBuilder); + String[][] input = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, + {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, + {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, + {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies + for (int i = 0; i < input.length; i++) { + client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(input[i]).endArray() + .endObject() + .endObject() + ) + .execute().actionGet(); + } + + client().prepareIndex(INDEX, PercolatorService.TYPE_NAME, "4") + .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) + .execute().actionGet(); + + refresh(); + + PercolateResponse response = client().preparePercolate().setIndices(INDEX).setDocumentType(TYPE) + .setGetRequest(Requests.getRequest(INDEX).type(TYPE).id("1")) + .execute().actionGet(); + assertThat(response.getCount(), equalTo(1l)); + } + + @Test + public void testBasicPrefixSuggestion() throws Exception { + completionMappingBuilder.payloads(true); + createIndexAndMapping(completionMappingBuilder); + for (int i = 0; i < 2; i++) { + createData(i == 0); + assertSuggestions("f", "Firestarter - The Prodigy", "Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters"); + assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro"); + assertSuggestions("ge", "Generator - Foo Fighters", "Get it on - Turbonegro"); + assertSuggestions("t", "The Prodigy", "Firestarter - The Prodigy", "Get it on - Turbonegro", "Turbonegro"); + } + } + + @Test + public void testThatWeightsAreWorking() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + List similarNames = Lists.newArrayList("the", "The Prodigy", "The Verve", "The the"); + // the weight is 1000 divided by string length, so the results are easy to to check + for (String similarName : similarNames) { + client().prepareIndex(INDEX, TYPE, similarName).setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(similarName).endArray() + .field("weight", 1000 / similarName.length()) + .endObject().endObject() + ).get(); + } + + refresh(); + + assertSuggestions("the", "the", "The the", "The Verve", "The Prodigy"); + } + + @Test + public void testThatWeightMustBeAnInteger() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + try { + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("sth").endArray() + .field("weight", 2.5) + .endObject().endObject() + ).get(); + fail("Indexing with a float weight was successful, but should not be"); + } catch (MapperParsingException e) { + assertThat(e.toString(), containsString("2.5")); + } + } + + @Test + public void testThatWeightCanBeAString() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("testing").endArray() + .field("weight", "10") + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("test").size(10) + ).execute().actionGet(); + + assertSuggestions(suggestResponse, "testSuggestions", "testing"); + Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); + assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); + CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; + + assertThat(prefixOption.getText().string(), equalTo("testing")); + assertThat((long) prefixOption.getScore(), equalTo(10l)); + } + + + @Test + public void testThatWeightMustNotBeANonNumberString() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + try { + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("sth").endArray() + .field("weight", "thisIsNotValid") + .endObject().endObject() + ).get(); + fail("Indexing with a non-number representing string as weight was successful, but should not be"); + } catch (MapperParsingException e) { + assertThat(e.toString(), containsString("thisIsNotValid")); + } + } + + @Test + public void testThatWeightAsStringMustBeInt() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + String weight = String.valueOf(Long.MAX_VALUE - 4); + try { + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("testing").endArray() + .field("weight", weight) + .endObject().endObject() + ).get(); + fail("Indexing with weight string representing value > Int.MAX_VALUE was successful, but should not be"); + } catch (MapperParsingException e) { + assertThat(e.toString(), containsString(weight)); + } + } + + @Test + public void testThatInputCanBeAStringInsteadOfAnArray() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .field("input", "Foo Fighters") + .field("output", "Boo Fighters") + .endObject().endObject() + ).get(); + + refresh(); + + assertSuggestions("f", "Boo Fighters"); + } + + @Test + public void testThatPayloadsAreArbitraryJsonObjects() throws Exception { + completionMappingBuilder.payloads(true); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .field("output", "Boo Fighters") + .startObject("payload").field("foo", "bar").startArray("test").value("spam").value("eggs").endArray().endObject() + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) + ).execute().actionGet(); + + assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); + Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); + assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); + CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; + assertThat(prefixOption.getPayload(), is(notNullValue())); + + // parse JSON + Map jsonMap = prefixOption.getPayloadAsMap(); + assertThat(jsonMap.size(), is(2)); + assertThat(jsonMap.get("foo").toString(), is("bar")); + assertThat(jsonMap.get("test"), is(instanceOf(List.class))); + List listValues = (List) jsonMap.get("test"); + assertThat(listValues, hasItems("spam", "eggs")); + } + + @Test + public void testPayloadAsNumeric() throws Exception { + completionMappingBuilder.payloads(true); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .field("output", "Boo Fighters") + .field("payload", 1) + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) + ).execute().actionGet(); + + assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); + Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); + assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); + CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; + assertThat(prefixOption.getPayload(), is(notNullValue())); + + assertThat(prefixOption.getPayloadAsLong(), equalTo(1l)); + } + + @Test + public void testPayloadAsString() throws Exception { + completionMappingBuilder.payloads(true); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .field("output", "Boo Fighters") + .field("payload", "test") + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + new CompletionSuggestionBuilder("testSuggestions").field(FIELD).text("foo").size(10) + ).execute().actionGet(); + + assertSuggestions(suggestResponse, "testSuggestions", "Boo Fighters"); + Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); + assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); + CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; + assertThat(prefixOption.getPayload(), is(notNullValue())); + + assertThat(prefixOption.getPayloadAsString(), equalTo("test")); + } + + @Test(expected = MapperException.class) + public void testThatExceptionIsThrownWhenPayloadsAreDisabledButInIndexRequest() throws Exception { + completionMappingBuilder.payloads(false); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .field("output", "Boo Fighters") + .startArray("payload").value("spam").value("eggs").endArray() + .endObject().endObject() + ).get(); + } + + @Test + public void testDisabledPreserveSeparators() throws Exception { + completionMappingBuilder.preserveSeparators(false); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .field("weight", 10) + .endObject().endObject() + ).get(); + + client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foof").endArray() + .field("weight", 20) + .endObject().endObject() + ).get(); + + refresh(); + + assertSuggestions("foof", "Foof", "Foo Fighters"); + } + + @Test + public void testEnabledPreserveSeparators() throws Exception { + completionMappingBuilder.preserveSeparators(true); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .endObject().endObject() + ).get(); + + client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foof").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + assertSuggestions("foof", "Foof"); + } + + @Test + public void testThatMultipleInputsAreSupported() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").value("Fu Fighters").endArray() + .field("output", "The incredible Foo Fighters") + .endObject().endObject() + ).get(); + + refresh(); + + assertSuggestions("foo", "The incredible Foo Fighters"); + assertSuggestions("fu", "The incredible Foo Fighters"); + } + + @Test + public void testThatShortSyntaxIsWorking() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startArray(FIELD) + .value("The Prodigy Firestarter").value("Firestarter") + .endArray().endObject() + ).get(); + + refresh(); + + assertSuggestions("t", "The Prodigy Firestarter"); + assertSuggestions("f", "Firestarter"); + } + + @Test + public void testThatDisablingPositionIncrementsWorkForStopwords() throws Exception { + // analyzer which removes stopwords... so may not be the simple one + completionMappingBuilder.searchAnalyzer("classic").indexAnalyzer("classic").preservePositionIncrements(false); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("The Beatles").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + assertSuggestions("b", "The Beatles"); + } + + @Test + public void testThatSynonymsWork() throws Exception { + Settings.Builder settingsBuilder = settingsBuilder() + .put("analysis.analyzer.suggest_analyzer_synonyms.type", "custom") + .put("analysis.analyzer.suggest_analyzer_synonyms.tokenizer", "standard") + .putArray("analysis.analyzer.suggest_analyzer_synonyms.filter", "standard", "lowercase", "my_synonyms") + .put("analysis.filter.my_synonyms.type", "synonym") + .putArray("analysis.filter.my_synonyms.synonyms", "foo,renamed"); + completionMappingBuilder.searchAnalyzer("suggest_analyzer_synonyms").indexAnalyzer("suggest_analyzer_synonyms"); + createIndexAndMappingAndSettings(settingsBuilder.build(), completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Foo Fighters").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + // get suggestions for renamed + assertSuggestions("r", "Foo Fighters"); + } + + @Test + public void testThatUpgradeToMultiFieldTypeWorks() throws Exception { + final XContentBuilder mapping = jsonBuilder() + .startObject() + .startObject(TYPE) + .startObject("properties") + .startObject(FIELD) + .field("type", "string") + .endObject() + .endObject() + .endObject() + .endObject(); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping(TYPE, mapping)); + client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); + ensureGreen(INDEX); + + PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "multi_field") + .startObject("fields") + .startObject(FIELD).field("type", "string").endObject() + .startObject("suggest").field("type", "completion").field("analyzer", "simple").endObject() + .endObject() + .endObject() + .endObject().endObject() + .endObject()) + .get(); + assertThat(putMappingResponse.isAcknowledged(), is(true)); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + new CompletionSuggestionBuilder("suggs").field(FIELD + ".suggest").text("f").size(10) + ).execute().actionGet(); + assertSuggestions(suggestResponse, "suggs"); + + client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); + ensureGreen(INDEX); + + SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldCompletionSuggestion("suggs").field(FIELD + ".suggest").text("f").size(10) + ).execute().actionGet(); + assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); + } + + @Test + public void testThatUpgradeToMultiFieldsWorks() throws Exception { + final XContentBuilder mapping = jsonBuilder() + .startObject() + .startObject(TYPE) + .startObject("properties") + .startObject(FIELD) + .field("type", "string") + .endObject() + .endObject() + .endObject() + .endObject(); + assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id))); + client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); + ensureGreen(INDEX); + + PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "string") + .startObject("fields") + .startObject("suggest").field("type", "completion").field("analyzer", "simple").endObject() + .endObject() + .endObject() + .endObject().endObject() + .endObject()) + .get(); + assertThat(putMappingResponse.isAcknowledged(), is(true)); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldCompletionSuggestion("suggs").field(FIELD + ".suggest").text("f").size(10) + ).execute().actionGet(); + assertSuggestions(suggestResponse, "suggs"); + + client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); + ensureGreen(INDEX); + + SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldCompletionSuggestion("suggs").field(FIELD + ".suggest").text("f").size(10) + ).execute().actionGet(); + assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); + } + + @Test + public void testThatFuzzySuggesterWorks() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Nirvana").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nirv").size(10) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + + suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nirw").size(10) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + } + + @Test + public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Nirvana").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + // edit distance 1 + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Norw").size(10) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo"); + + // edit distance 2 + suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Norw").size(10).setFuzziness(Fuzziness.TWO) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + } + + @Test + public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Nirvana").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(false).setFuzziness(Fuzziness.ONE) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo"); + + suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(true).setFuzziness(Fuzziness.ONE) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + } + + @Test + public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Nirvana").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nriva").size(10).setFuzzyMinLength(6) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo"); + + suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nrivan").size(10).setFuzzyMinLength(6) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + } + + @Test + public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Nirvana").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nirw").size(10).setFuzzyPrefixLength(4) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo"); + + suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("Nirvo").size(10).setFuzzyPrefixLength(4) + ).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + } + + @Test + public void testThatFuzzySuggesterIsUnicodeAware() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("ööööö").endArray() + .endObject().endObject() + ).get(); + + refresh(); + + // suggestion with a character, which needs unicode awareness + CompletionSuggestionFuzzyBuilder completionSuggestionBuilder = + SuggestBuilders.oldFuzzyCompletionSuggestion("foo").field(FIELD).text("öööи").size(10).setUnicodeAware(true); + + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "ööööö"); + + // removing unicode awareness leads to no result + completionSuggestionBuilder.setUnicodeAware(false); + suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo"); + + // increasing edit distance instead of unicode awareness works again, as this is only a single character + completionSuggestionBuilder.setFuzziness(Fuzziness.TWO); + suggestResponse = client().prepareSuggest(INDEX).addSuggestion(completionSuggestionBuilder).execute().actionGet(); + assertSuggestions(suggestResponse, false, "foo", "ööööö"); + } + + @Test + public void testThatStatsAreWorking() throws Exception { + String otherField = "testOtherField"; + + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id))); + + PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD.toString()) + .field("type", "completion").field("analyzer", "simple") + .endObject() + .startObject(otherField) + .field("type", "completion").field("analyzer", "simple") + .endObject() + .endObject().endObject().endObject()) + .get(); + assertThat(putMappingResponse.isAcknowledged(), is(true)); + + // Index two entities + client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()).get(); + client().prepareIndex(INDEX, TYPE, "2").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()).get(); + + // Get all stats + IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get(); + CompletionStats completionStats = indicesStatsResponse.getIndex(INDEX).getPrimaries().completion; + assertThat(completionStats, notNullValue()); + long totalSizeInBytes = completionStats.getSizeInBytes(); + assertThat(totalSizeInBytes, is(greaterThan(0L))); + + IndicesStatsResponse singleFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields(FIELD).get(); + long singleFieldSizeInBytes = singleFieldStats.getIndex(INDEX).getPrimaries().completion.getFields().get(FIELD); + IndicesStatsResponse otherFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields(otherField).get(); + long otherFieldSizeInBytes = otherFieldStats.getIndex(INDEX).getPrimaries().completion.getFields().get(otherField); + assertThat(singleFieldSizeInBytes + otherFieldSizeInBytes, is(totalSizeInBytes)); + + // regexes + IndicesStatsResponse regexFieldStats = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).setCompletionFields("*").get(); + ObjectLongHashMap fields = regexFieldStats.getIndex(INDEX).getPrimaries().completion.getFields(); + long regexSizeInBytes = fields.get(FIELD) + fields.get(otherField); + assertThat(regexSizeInBytes, is(totalSizeInBytes)); + } + + @Test + public void testThatSortingOnCompletionFieldReturnsUsefulException() throws Exception { + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Nirvana").endArray() + .endObject().endObject() + ).get(); + + refresh(); + try { + client().prepareSearch(INDEX).setTypes(TYPE).addSort(new FieldSortBuilder(FIELD)).execute().actionGet(); + fail("Expected an exception due to trying to sort on completion field, but did not happen"); + } catch (SearchPhaseExecutionException e) { + assertThat(e.status().getStatus(), is(400)); + assertThat(e.toString(), containsString("Sorting not supported for field[" + FIELD + "]")); + } + } + + @Test + public void testThatSuggestStopFilterWorks() throws Exception { + Settings.Builder settingsBuilder = settingsBuilder() + .put("index.analysis.analyzer.stoptest.tokenizer", "standard") + .putArray("index.analysis.analyzer.stoptest.filter", "standard", "suggest_stop_filter") + .put("index.analysis.filter.suggest_stop_filter.type", "stop") + .put("index.analysis.filter.suggest_stop_filter.remove_trailing", false); + + CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); + completionMappingBuilder.preserveSeparators(true).preservePositionIncrements(true); + completionMappingBuilder.searchAnalyzer("stoptest"); + completionMappingBuilder.indexAnalyzer("simple"); + createIndexAndMappingAndSettings(settingsBuilder.build(), completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Feed trolls").endArray() + .field("weight", 5).endObject().endObject() + ).get(); + + // Higher weight so it's ranked first: + client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("Feed the trolls").endArray() + .field("weight", 10).endObject().endObject() + ).get(); + + refresh(); + + assertSuggestions("f", "Feed the trolls", "Feed trolls"); + assertSuggestions("fe", "Feed the trolls", "Feed trolls"); + assertSuggestions("fee", "Feed the trolls", "Feed trolls"); + assertSuggestions("feed", "Feed the trolls", "Feed trolls"); + assertSuggestions("feed t", "Feed the trolls", "Feed trolls"); + assertSuggestions("feed the", "Feed the trolls"); + // stop word complete, gets ignored on query time, makes it "feed" only + assertSuggestions("feed the ", "Feed the trolls", "Feed trolls"); + // stopword gets removed, but position increment kicks in, which doesnt work for the prefix suggester + assertSuggestions("feed the t"); + } + + @Test(expected = MapperParsingException.class) + public void testThatIndexingInvalidFieldsInCompletionFieldResultsInException() throws Exception { + CompletionMappingBuilder completionMappingBuilder = new CompletionMappingBuilder(); + createIndexAndMapping(completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("FRIGGININVALID").value("Nirvana").endArray() + .endObject().endObject()).get(); + } + + + public void assertSuggestions(String suggestion, String... suggestions) { + String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggestion).size(10) + ).execute().actionGet(); + + assertSuggestions(suggestResponse, suggestionName, suggestions); + } + + public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) { + String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10); + SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion( + SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggestString).size(10) + ).execute().actionGet(); + + assertSuggestions(suggestResponse, false, suggestionName, suggestions); + } + + private void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { + assertSuggestions(suggestResponse, true, name, suggestions); + } + + private void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { + assertAllSuccessful(suggestResponse); + + List suggestionNames = Lists.newArrayList(); + for (Suggest.Suggestion> suggestion : Lists.newArrayList(suggestResponse.getSuggest().iterator())) { + suggestionNames.add(suggestion.getName()); + } + String expectFieldInResponseMsg = String.format(Locale.ROOT, "Expected suggestion named %s in response, got %s", name, suggestionNames); + assertThat(expectFieldInResponseMsg, suggestResponse.getSuggest().getSuggestion(name), is(notNullValue())); + + Suggest.Suggestion> suggestion = suggestResponse.getSuggest().getSuggestion(name); + + List suggestionList = getNames(suggestion.getEntries().get(0)); + List options = suggestion.getEntries().get(0).getOptions(); + + String assertMsg = String.format(Locale.ROOT, "Expected options %s length to be %s, but was %s", suggestionList, suggestions.length, options.size()); + assertThat(assertMsg, options.size(), is(suggestions.length)); + if (suggestionOrderStrict) { + for (int i = 0; i < suggestions.length; i++) { + String errMsg = String.format(Locale.ROOT, "Expected elem %s in list %s to be [%s] score: %s", i, suggestionList, suggestions[i], options.get(i).getScore()); + assertThat(errMsg, options.get(i).getText().toString(), is(suggestions[i])); + } + } else { + for (String expectedSuggestion : suggestions) { + String errMsg = String.format(Locale.ROOT, "Expected elem %s to be in list %s", expectedSuggestion, suggestionList); + assertThat(errMsg, suggestionList, hasItem(expectedSuggestion)); + } + } + } + + private List getNames(Suggest.Suggestion.Entry suggestEntry) { + List names = Lists.newArrayList(); + for (Suggest.Suggestion.Entry.Option entry : suggestEntry.getOptions()) { + names.add(entry.getText().string()); + } + return names; + } + + private void createIndexAndMappingAndSettings(Settings settings, CompletionMappingBuilder completionMappingBuilder) throws IOException { + assertAcked(client().admin().indices().prepareCreate(INDEX) + .setSettings(Settings.settingsBuilder().put(indexSettings()).put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .field("analyzer", completionMappingBuilder.indexAnalyzer) + .field("search_analyzer", completionMappingBuilder.searchAnalyzer) + .field("payloads", completionMappingBuilder.payloads) + .field("preserve_separators", completionMappingBuilder.preserveSeparators) + .field("preserve_position_increments", completionMappingBuilder.preservePositionIncrements) + .endObject() + .endObject().endObject() + .endObject()) + .get()); + ensureYellow(); + } + + private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException { + createIndexAndMappingAndSettings(Settings.EMPTY, completionMappingBuilder); + } + + private void createData(boolean optimize) throws IOException, InterruptedException, ExecutionException { + String[][] input = {{"Foo Fighters"}, {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; + String[] surface = {"Foo Fighters", "Generator - Foo Fighters", "Learn to Fly - Foo Fighters", "The Prodigy", "Firestarter - The Prodigy", "Turbonegro", "Get it on - Turbonegro"}; + int[] weight = {10, 9, 8, 12, 11, 6, 7}; + IndexRequestBuilder[] builders = new IndexRequestBuilder[input.length]; + for (int i = 0; i < builders.length; i++) { + builders[i] = client().prepareIndex(INDEX, TYPE, "" + i) + .setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(input[i]).endArray() + .field("output", surface[i]) + .startObject("payload").field("id", i).endObject() + .field("weight", 1) // WE FORCEFULLY INDEX A BOGUS WEIGHT + .endObject() + .endObject() + ); + } + indexRandom(false, builders); + + for (int i = 0; i < builders.length; i++) { // add them again to make sure we deduplicate on the surface form + builders[i] = client().prepareIndex(INDEX, TYPE, "n" + i) + .setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(input[i]).endArray() + .field("output", surface[i]) + .startObject("payload").field("id", i).endObject() + .field("weight", weight[i]) + .endObject() + .endObject() + ); + } + indexRandom(false, builders); + + client().admin().indices().prepareRefresh(INDEX).execute().actionGet(); + if (optimize) { + // make sure merging works just fine + client().admin().indices().prepareFlush(INDEX).execute().actionGet(); + client().admin().indices().prepareOptimize(INDEX).setMaxNumSegments(randomIntBetween(1, 5)).get(); + } + } + + @Test // see #3555 + public void testPrunedSegments() throws IOException { + createIndexAndMappingAndSettings(settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build(), completionMappingBuilder); + + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value("The Beatles").endArray() + .endObject().endObject() + ).get(); + client().prepareIndex(INDEX, TYPE, "2").setSource(jsonBuilder() + .startObject() + .field("somefield", "somevalue") + .endObject() + ).get(); // we have 2 docs in a segment... + OptimizeResponse actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).execute().actionGet(); + assertAllSuccessful(actionGet); + refresh(); + // update the first one and then merge.. the target segment will have no value in FIELD + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject() + .field("somefield", "somevalue") + .endObject() + ).get(); + actionGet = client().admin().indices().prepareOptimize().setFlush(true).setMaxNumSegments(1).execute().actionGet(); + assertAllSuccessful(actionGet); + refresh(); + + assertSuggestions("b"); + assertThat(2l, equalTo(client().prepareCount(INDEX).get().getCount())); + for (IndexShardSegments seg : client().admin().indices().prepareSegments().get().getIndices().get(INDEX)) { + ShardSegments[] shards = seg.getShards(); + for (ShardSegments shardSegments : shards) { + assertThat(shardSegments.getSegments().size(), equalTo(1)); + } + } + } + + @Test + public void testMaxFieldLength() throws IOException { + client().admin().indices().prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).get(); + ensureGreen(); + int iters = scaledRandomIntBetween(10, 20); + for (int i = 0; i < iters; i++) { + int maxInputLen = between(3, 50); + String str = replaceReservedChars(randomRealisticUnicodeOfCodepointLengthBetween(maxInputLen + 1, maxInputLen + scaledRandomIntBetween(2, 50)), (char) 0x01); + assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .field("max_input_length", maxInputLen) + // upgrade mapping each time + .field("analyzer", "keyword") + .endObject() + .endObject().endObject() + .endObject())); + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(str).endArray() + .field("output", "foobar") + .endObject().endObject() + ).setRefresh(true).get(); + // need to flush and refresh, because we keep changing the same document + // we have to make sure that segments without any live documents are deleted + flushAndRefresh(); + int prefixLen = OldCompletionFieldMapper.correctSubStringLen(str, between(1, maxInputLen - 1)); + assertSuggestions(str.substring(0, prefixLen), "foobar"); + if (maxInputLen + 1 < str.length()) { + int offset = Character.isHighSurrogate(str.charAt(maxInputLen - 1)) ? 2 : 1; + int correctSubStringLen = OldCompletionFieldMapper.correctSubStringLen(str, maxInputLen + offset); + String shortenedSuggestion = str.substring(0, correctSubStringLen); + assertSuggestions(shortenedSuggestion); + } + } + } + + @Test + // see #3596 + public void testVeryLongInput() throws IOException { + assertAcked(client().admin().indices().prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .endObject() + .endObject().endObject() + .endObject()).get()); + ensureYellow(); + // can cause stack overflow without the default max_input_length + String longString = replaceReservedChars(randomRealisticUnicodeOfLength(randomIntBetween(5000, 10000)), (char) 0x01); + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(longString).endArray() + .field("output", "foobar") + .endObject().endObject() + ).setRefresh(true).get(); + + } + + // see #3648 + @Test(expected = MapperParsingException.class) + public void testReservedChars() throws IOException { + assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .endObject() + .endObject().endObject() + .endObject()).get()); + ensureYellow(); + // can cause stack overflow without the default max_input_length + String string = "foo" + (char) 0x00 + "bar"; + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject().startObject(FIELD) + .startArray("input").value(string).endArray() + .field("output", "foobar") + .endObject().endObject() + ).setRefresh(true).get(); + } + + @Test // see #5930 + public void testIssue5930() throws IOException { + assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, jsonBuilder().startObject() + .startObject(TYPE).startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .endObject() + .endObject().endObject() + .endObject()).get()); + ensureYellow(); + String string = "foo bar"; + client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder() + .startObject() + .field(FIELD, string) + .endObject() + ).setRefresh(true).get(); + + try { + client().prepareSearch(INDEX).addAggregation(AggregationBuilders.terms("suggest_agg").field(FIELD) + .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet(); + // Exception must be thrown + assertFalse(true); + } catch (SearchPhaseExecutionException e) { + assertTrue(e.toString().contains("found no fielddata type for field [" + FIELD + "]")); + } + } + + // see issue #6399 + @Test + public void testIndexingUnrelatedNullValue() throws Exception { + String mapping = jsonBuilder() + .startObject() + .startObject(TYPE) + .startObject("properties") + .startObject(FIELD) + .field("type", "completion") + .endObject() + .endObject() + .endObject() + .endObject() + .string(); + + assertAcked(client().admin().indices().prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping(TYPE, mapping).get()); + ensureGreen(); + + client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "strings make me happy", FIELD + "_1", "nulls make me sad") + .setRefresh(true).get(); + + try { + client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad") + .setRefresh(true).get(); + fail("Expected MapperParsingException for null value"); + } catch (MapperParsingException e) { + // make sure that the exception has the name of the field causing the error + assertTrue(e.getDetailedMessage().contains(FIELD)); + } + + } + + private static String replaceReservedChars(String input, char replacement) { + char[] charArray = input.toCharArray(); + for (int i = 0; i < charArray.length; i++) { + if (OldCompletionFieldMapper.isReservedChar(charArray[i])) { + charArray[i] = replacement; + } + } + return new String(charArray); + } + + private static class CompletionMappingBuilder { + private String searchAnalyzer = "simple"; + private String indexAnalyzer = "simple"; + private Boolean payloads = getRandom().nextBoolean(); + private Boolean preserveSeparators = getRandom().nextBoolean(); + private Boolean preservePositionIncrements = getRandom().nextBoolean(); + + public CompletionMappingBuilder searchAnalyzer(String searchAnalyzer) { + this.searchAnalyzer = searchAnalyzer; + return this; + } + public CompletionMappingBuilder indexAnalyzer(String indexAnalyzer) { + this.indexAnalyzer = indexAnalyzer; + return this; + } + public CompletionMappingBuilder payloads(Boolean payloads) { + this.payloads = payloads; + return this; + } + public CompletionMappingBuilder preserveSeparators(Boolean preserveSeparators) { + this.preserveSeparators = preserveSeparators; + return this; + } + public CompletionMappingBuilder preservePositionIncrements(Boolean preservePositionIncrements) { + this.preservePositionIncrements = preservePositionIncrements; + return this; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/OldContextSuggestSearchIT.java similarity index 86% rename from core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchIT.java rename to core/src/test/java/org/elasticsearch/search/suggest/OldContextSuggestSearchIT.java index 83c8b362d8bab..9f6aa161668ff 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/OldContextSuggestSearchIT.java @@ -20,25 +20,29 @@ import com.google.common.collect.Sets; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.search.suggest.Suggest.Suggestion; import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry; import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option; -import org.elasticsearch.search.suggest.completion.CompletionSuggestion; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder; -import org.elasticsearch.search.suggest.context.ContextBuilder; -import org.elasticsearch.search.suggest.context.ContextMapping; -import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestion; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestionBuilder; +import org.elasticsearch.search.suggest.completion.old.CompletionSuggestionFuzzyBuilder; +import org.elasticsearch.search.suggest.completion.old.context.ContextBuilder; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; import org.junit.Test; @@ -51,11 +55,12 @@ import static org.hamcrest.Matchers.containsString; @SuppressCodecs("*") // requires custom completion format -public class ContextSuggestSearchIT extends ESIntegTestCase { +public class OldContextSuggestSearchIT extends ESIntegTestCase { private static final String INDEX = "test"; private static final String TYPE = "testType"; private static final String FIELD = "testField"; + private final Version PRE2X_VERSION = VersionUtils.randomVersionBetween(getRandom(), Version.V_1_0_0, Version.V_1_7_0); private static final String[][] HEROS = { { "Afari, Jamal", "Jamal Afari", "Jamal" }, @@ -78,7 +83,8 @@ public class ContextSuggestSearchIT extends ESIntegTestCase { @Test public void testBasicGeo() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st").precision("5km").neighbors(true)))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st").precision("5km").neighbors(true)))); ensureYellow(); XContentBuilder source1 = jsonBuilder() @@ -104,7 +110,7 @@ public void testBasicGeo() throws Exception { client().admin().indices().prepareRefresh(INDEX).get(); String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text("h").size(10) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text("h").size(10) .addGeoLocation("st", 52.52, 13.4); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); @@ -116,20 +122,21 @@ public void testBasicGeo() throws Exception { @Test public void testMultiLevelGeo() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st") - .precision(1) - .precision(2) - .precision(3) - .precision(4) - .precision(5) - .precision(6) - .precision(7) - .precision(8) - .precision(9) - .precision(10) - .precision(11) - .precision(12) - .neighbors(true)))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st") + .precision(1) + .precision(2) + .precision(3) + .precision(4) + .precision(5) + .precision(6) + .precision(7) + .precision(8) + .precision(9) + .precision(10) + .precision(11) + .precision(12) + .neighbors(true)))); ensureYellow(); XContentBuilder source1 = jsonBuilder() @@ -175,7 +182,7 @@ public void testMappingIdempotency() throws Exception { .endObject().endObject() .endObject().endObject(); - assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping.string())); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping(TYPE, mapping.string())); ensureYellow(); Collections.shuffle(precisions, getRandom()); @@ -216,7 +223,7 @@ public void testGeoField() throws Exception { mapping.endObject(); mapping.endObject(); - assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping(TYPE, mapping)); ensureYellow(); XContentBuilder source1 = jsonBuilder() @@ -244,7 +251,7 @@ public void testGeoField() throws Exception { refresh(); String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text("h").size(10) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text("h").size(10) .addGeoLocation("st", 52.52, 13.4); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); @@ -270,7 +277,8 @@ public void testSimpleGeo() throws Exception { double precision = 100.0; // meters - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st").precision(precision).neighbors(true)))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.location("st").precision(precision).neighbors(true)))); ensureYellow(); String[] locations = { reinickendorf, pankow, koepenick, bernau, berlin, mitte, steglitz, wilmersdorf, spandau, tempelhof, @@ -310,7 +318,8 @@ public void testSimpleGeo() throws Exception { @Test public void testSimplePrefix() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.category("st")))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.category("st")))); ensureYellow(); for (int i = 0; i < HEROS.length; i++) { @@ -342,7 +351,7 @@ public void testTypeCategoryIsActuallyCalledCategory() throws Exception { .startObject("context").startObject("color").field("type", "category").endObject().endObject() .endObject() .endObject().endObject().endObject(); - assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping(TYPE, mapping)); ensureYellow(); XContentBuilder doc1 = jsonBuilder(); doc1.startObject().startObject("suggest_field") @@ -372,7 +381,7 @@ public void testTypeCategoryIsActuallyCalledCategory() throws Exception { private void getBackpackSuggestionAndCompare(String contextValue, String... expectedText) { Set expected = Sets.newHashSet(expectedText); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion("suggestion").field("suggest_field").text("back").size(10).addContextField("color", contextValue); + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest_field").text("back").size(10).addContextField("color", contextValue); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); Suggest suggest = suggestResponse.getSuggest(); @@ -393,7 +402,8 @@ private void getBackpackSuggestionAndCompare(String contextValue, String... expe @Test public void testBasic() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, false, ContextBuilder.reference("st", "_type"), ContextBuilder.reference("nd", "_type")))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, false, ContextBuilder.reference("st", "_type"), ContextBuilder.reference("nd", "_type")))); ensureYellow(); client().prepareIndex(INDEX, TYPE, "1") @@ -410,7 +420,8 @@ public void testBasic() throws Exception { @Test public void testSimpleField() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "category")))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "category")))); ensureYellow(); for (int i = 0; i < HEROS.length; i++) { @@ -455,7 +466,8 @@ public void testEmptySuggestion() throws Exception { .endObject() .string(); - assertAcked(client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, mapping).get()); + assertAcked(client().admin().indices().prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, mapping).get()); ensureGreen(); client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "") @@ -465,7 +477,8 @@ public void testEmptySuggestion() throws Exception { @Test public void testMultiValueField() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "category")))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "category")))); ensureYellow(); for (int i = 0; i < HEROS.length; i++) { @@ -491,7 +504,8 @@ public void testMultiValueField() throws Exception { @Test public void testMultiContext() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "categoryA"), ContextBuilder.reference("nd", "categoryB")))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "categoryA"), ContextBuilder.reference("nd", "categoryB")))); ensureYellow(); for (int i = 0; i < HEROS.length; i++) { @@ -518,7 +532,8 @@ public void testMultiContext() throws Exception { @Test public void testMultiContextWithFuzzyLogic() throws Exception { - assertAcked(prepareCreate(INDEX).addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "categoryA"), ContextBuilder.reference("nd", "categoryB")))); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) + .addMapping(TYPE, createMapping(TYPE, ContextBuilder.reference("st", "categoryA"), ContextBuilder.reference("nd", "categoryB")))); ensureYellow(); for (int i = 0; i < HEROS.length; i++) { @@ -551,7 +566,7 @@ public void testMultiContextWithFuzzyLogic() throws Exception { public void testSimpleType() throws Exception { String[] types = { TYPE + "A", TYPE + "B", TYPE + "C" }; - CreateIndexRequestBuilder createIndexRequestBuilder = prepareCreate(INDEX); + CreateIndexRequestBuilder createIndexRequestBuilder = prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)); for (String type : types) { createIndexRequestBuilder.addMapping(type, createMapping(type, ContextBuilder.reference("st", "_type"))); } @@ -594,13 +609,13 @@ public void testGeoContextDefaultMapping() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("poi", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("poi", xContentBuilder)); ensureYellow(); index(INDEX, "poi", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Alexanderplatz").endObject().endObject()); refresh(); - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", berlinAlexanderplatz.lat(), berlinAlexanderplatz.lon()); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", berlinAlexanderplatz.lat(), berlinAlexanderplatz.lon()); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz"); } @@ -617,7 +632,7 @@ public void testThatMissingPrefixesForContextReturnException() throws Exception .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("service", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("service", xContentBuilder)); ensureYellow(); // now index a document with color field @@ -644,7 +659,7 @@ public void testThatLocationVenueCanBeParsedAsDocumented() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("poi", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("poi", xContentBuilder)); ensureYellow(); SuggestRequest suggestRequest = new SuggestRequest(INDEX); @@ -674,14 +689,14 @@ public void testThatCategoryDefaultWorks() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("item", xContentBuilder)); ensureYellow(); index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie red").endObject().endObject()); index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie blue").startObject("context").field("color", "blue").endObject().endObject().endObject()); refresh(); - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("h").size(10).addContextField("color", "red"); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest").text("h").size(10).addContextField("color", "red"); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Hoodie red"); } @@ -699,14 +714,14 @@ public void testThatDefaultCategoryAndPathWorks() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("item", xContentBuilder)); ensureYellow(); index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie red").endObject().endObject()); index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Hoodie blue").endObject().field("color", "blue").endObject()); refresh(); - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("h").size(10).addContextField("color", "red"); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest").text("h").size(10).addContextField("color", "red"); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Hoodie red"); } @@ -723,7 +738,7 @@ public void testThatGeoPrecisionIsWorking() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("item", xContentBuilder)); ensureYellow(); // lets create some locations by geohashes in different cells with the precision 4 @@ -739,7 +754,7 @@ public void testThatGeoPrecisionIsWorking() throws Exception { index(INDEX, "item", "4", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Dahlem").field("weight", 1).startObject("context").startObject("location").field("lat", dahlem.lat()).field("lon", dahlem.lon()).endObject().endObject().endObject().endObject()); refresh(); - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz", "Berlin Poelchaustr.", "Berlin Dahlem"); } @@ -757,7 +772,7 @@ public void testThatNeighborsCanBeExcluded() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("item", xContentBuilder)); ensureYellow(); GeoPoint alexanderplatz = GeoHashUtils.decode("u33dc1"); @@ -769,7 +784,7 @@ public void testThatNeighborsCanBeExcluded() throws Exception { index(INDEX, "item", "2", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Hackescher Markt").field("weight", 2).startObject("context").startObject("location").field("lat", cellNeighbourOfAlexanderplatz.lat()).field("lon", cellNeighbourOfAlexanderplatz.lon()).endObject().endObject().endObject().endObject()); refresh(); - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz"); } @@ -787,14 +802,14 @@ public void testThatGeoPathCanBeSelected() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("item", xContentBuilder)); ensureYellow(); GeoPoint alexanderplatz = GeoHashUtils.decode("u33dc1"); index(INDEX, "item", "1", jsonBuilder().startObject().startObject("suggest").field("input", "Berlin Alexanderplatz").endObject().startObject("loc").field("lat", alexanderplatz.lat()).field("lon", alexanderplatz.lon()).endObject().endObject()); refresh(); - CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); + CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.oldCompletionSuggestion("suggestion").field("suggest").text("b").size(10).addGeoLocation("location", alexanderplatz.lat(), alexanderplatz.lon()); SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionBuilder).get(); assertSuggestion(suggestResponse.getSuggest(), 0, "suggestion", "Berlin Alexanderplatz"); } @@ -811,7 +826,7 @@ public void testThatPrecisionIsRequired() throws Exception { .endObject().endObject().endObject() .endObject(); - assertAcked(prepareCreate(INDEX).addMapping("item", xContentBuilder)); + assertAcked(prepareCreate(INDEX).setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).addMapping("item", xContentBuilder)); } @Test @@ -826,7 +841,7 @@ public void testThatLatLonParsingFromSourceWorks() throws Exception { .endObject().endObject().endObject() .endObject().endObject(); - assertAcked(prepareCreate("test").setSource(xContentBuilder.bytes())); + assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)).setSource(xContentBuilder.bytes())); double latitude = 52.22; double longitude = 4.53; @@ -852,7 +867,7 @@ public void testThatLatLonParsingFromSourceWorks() throws Exception { public void assertGeoSuggestionsInRange(String location, String suggest, double precision) throws IOException { String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) .addGeoLocation("st", location); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); @@ -875,7 +890,7 @@ public void assertGeoSuggestionsInRange(String location, String suggest, double public void assertPrefixSuggestions(long prefix, String suggest, String... hits) throws IOException { String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggest) .size(hits.length + 1).addCategory("st", Long.toString(prefix)); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); @@ -900,7 +915,7 @@ public void assertPrefixSuggestions(long prefix, String suggest, String... hits) public void assertContextWithFuzzySuggestions(String[] prefix1, String[] prefix2, String suggest, String... hits) throws IOException { String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionFuzzyBuilder context = SuggestBuilders.fuzzyCompletionSuggestion(suggestionName).field(FIELD).text(suggest) + CompletionSuggestionFuzzyBuilder context = SuggestBuilders.oldFuzzyCompletionSuggestion(suggestionName).field(FIELD).text(suggest) .size(hits.length + 10).addContextField("st", prefix1).addContextField("nd", prefix2).setFuzziness(Fuzziness.TWO); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); @@ -928,7 +943,7 @@ public void assertContextWithFuzzySuggestions(String[] prefix1, String[] prefix2 public void assertFieldSuggestions(String value, String suggest, String... hits) throws IOException { String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) .addContextField("st", value); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); @@ -953,7 +968,7 @@ public void assertFieldSuggestions(String value, String suggest, String... hits) public void assertDoubleFieldSuggestions(String field1, String field2, String suggest, String... hits) throws IOException { String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) .addContextField("st", field1).addContextField("nd", field2); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); SuggestResponse suggestResponse = suggestionRequest.execute().actionGet(); @@ -977,7 +992,7 @@ public void assertDoubleFieldSuggestions(String field1, String field2, String su public void assertMultiContextSuggestions(String value1, String value2, String suggest, String... hits) throws IOException { String suggestionName = randomAsciiOfLength(10); - CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) + CompletionSuggestionBuilder context = SuggestBuilders.oldCompletionSuggestion(suggestionName).field(FIELD).text(suggest).size(10) .addContextField("st", value1).addContextField("nd", value2); SuggestRequestBuilder suggestionRequest = client().prepareSuggest(INDEX).addSuggestion(context); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTest.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTest.java new file mode 100644 index 0000000000000..6aa569998feae --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTest.java @@ -0,0 +1,314 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.suggest.xdocument.ContextSuggestField; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.index.mapper.*; +import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping; +import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext; +import org.elasticsearch.search.suggest.completion.context.ContextBuilder; +import org.elasticsearch.search.suggest.completion.context.ContextMapping; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.Test; + +import java.util.*; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isIn; + +public class CategoryContextMappingTest extends ESSingleNodeTestCase { + + @Test + public void testIndexingWithNoContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .array("input", "suggestion1", "suggestion2") + .field("weight", 3) + .endObject() + .startObject() + .array("input", "suggestion3", "suggestion4") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 7); + } + + @Test + public void testIndexingWithSimpleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .field("ctx", "ctx1") + .endObject() + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + @Test + public void testIndexingWithContextList() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startObject("completion") + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .array("ctx", "ctx1", "ctx2", "ctx3") + .endObject() + .field("weight", 5) + .endObject() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + @Test + public void testIndexingWithMultipleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "category") + .endObject() + .startObject() + .field("name", "type") + .field("type", "category") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + XContentBuilder builder = jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .startObject("contexts") + .array("ctx", "ctx1", "ctx2", "ctx3") + .array("type", "typr3", "ftg") + .endObject() + .endObject() + .endArray() + .endObject(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", builder.bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 6); + } + + @Test + public void testQueryContextParsingBasic() throws Exception { + XContentBuilder builder = jsonBuilder().value("context1"); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("cat", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + CategoryQueryContext queryContext = iterator.next(); + assertThat(queryContext.context.toString(), equalTo("context1")); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.isPrefix, equalTo(false)); + } + + @Test + public void testQueryContextParsingArray() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .value("context1") + .value("context2") + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("cat", parser); + List expectedContexts = new ArrayList<>(Arrays.asList("context1", "context2")); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + CategoryQueryContext queryContext = iterator.next(); + assertThat(queryContext.context.toString(), isIn(expectedContexts)); + assertTrue(iterator.hasNext()); + queryContext = iterator.next(); + assertThat(queryContext.context.toString(), isIn(expectedContexts)); + } + + @Test + public void testQueryContextParsingObject() throws Exception { + XContentBuilder builder = jsonBuilder().startObject() + .field("context", "context1") + .field("boost", 10) + .field("prefix", true) + .endObject(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("cat", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + CategoryQueryContext queryContext = iterator.next(); + assertThat(queryContext.context.toString(), equalTo("context1")); + assertThat(queryContext.boost, equalTo(10)); + assertThat(queryContext.isPrefix, equalTo(true)); + } + + + @Test + public void testQueryContextParsingObjectArray() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .field("context", "context1") + .field("boost", 2) + .field("prefix", true) + .endObject() + .startObject() + .field("context", "context2") + .field("boost", 3) + .field("prefix", false) + .endObject() + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("cat", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + CategoryQueryContext queryContext = iterator.next(); + assertThat(queryContext.context.toString(), equalTo("context1")); + assertThat(queryContext.boost, equalTo(2)); + assertThat(queryContext.isPrefix, equalTo(true)); + assertTrue(iterator.hasNext()); + queryContext = iterator.next(); + assertThat(queryContext.context.toString(), equalTo("context2")); + assertThat(queryContext.boost, equalTo(3)); + assertThat(queryContext.isPrefix, equalTo(false)); + } + + @Test + public void testQueryContextParsingMixed() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .field("context", "context1") + .field("boost", 2) + .field("prefix", true) + .endObject() + .value("context2") + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + CategoryContextMapping mapping = ContextBuilder.category("cat").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("cat", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + CategoryQueryContext queryContext = iterator.next(); + assertThat(queryContext.context.toString(), equalTo("context1")); + assertThat(queryContext.boost, equalTo(2)); + assertThat(queryContext.isPrefix, equalTo(true)); + assertTrue(iterator.hasNext()); + queryContext = iterator.next(); + assertThat(queryContext.context.toString(), equalTo("context2")); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.isPrefix, equalTo(false)); + } + + @Test + public void testParsingContextFromDocument() throws Exception { + CategoryContextMapping mapping = ContextBuilder.category("cat").field("category").build(); + ParseContext.Document document = new ParseContext.Document(); + document.add(new StringField("category", "category1", Field.Store.NO)); + Set context = mapping.parseContext(document); + assertThat(context.size(), equalTo(1)); + assertTrue(context.contains("category1")); + } + + static void assertContextSuggestFields(IndexableField[] fields, int expected) { + int actualFieldCount = 0; + for (IndexableField field : fields) { + if (field instanceof ContextSuggestField) { + actualFieldCount++; + } + } + assertThat(actualFieldCount, equalTo(expected)); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTest.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTest.java new file mode 100644 index 0000000000000..6f2386ac9b839 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTest.java @@ -0,0 +1,317 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.suggest.completion; + +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.search.suggest.completion.context.*; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.Test; + +import java.util.*; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.search.suggest.completion.CategoryContextMappingTest.assertContextSuggestFields; +import static org.hamcrest.Matchers.equalTo; + +public class GeoContextMappingTest extends ESSingleNodeTestCase { + + @Test + public void testIndexingWithNoContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "geo") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .array("input", "suggestion1", "suggestion2") + .field("weight", 3) + .endObject() + .startObject() + .array("input", "suggestion3", "suggestion4") + .field("weight", 4) + .endObject() + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 7); + } + + @Test + public void testIndexingWithSimpleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "geo") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .startObject("ctx") + .field("lat", 43.6624803) + .field("lon", -79.3863353) + .endObject() + .endObject() + .field("weight", 5) + .endObject() + .endArray() + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + @Test + public void testIndexingWithContextList() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "ctx") + .field("type", "geo") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder() + .startObject() + .startObject("completion") + .field("input", "suggestion5", "suggestion6", "suggestion7") + .startObject("contexts") + .startArray("ctx") + .startObject() + .field("lat", 43.6624803) + .field("lon", -79.3863353) + .endObject() + .startObject() + .field("lat", 43.6624718) + .field("lon", -79.3873227) + .endObject() + .endArray() + .endObject() + .field("weight", 5) + .endObject() + .bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 3); + } + + @Test + public void testIndexingWithMultipleContexts() throws Exception { + String mapping = jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("completion") + .field("type", "completion") + .startArray("contexts") + .startObject() + .field("name", "loc1") + .field("type", "geo") + .endObject() + .startObject() + .field("name", "loc2") + .field("type", "geo") + .endObject() + .endArray() + .endObject().endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + FieldMapper fieldMapper = defaultMapper.mappers().getMapper("completion"); + MappedFieldType completionFieldType = fieldMapper.fieldType(); + XContentBuilder builder = jsonBuilder() + .startObject() + .startArray("completion") + .startObject() + .field("input", "suggestion5", "suggestion6", "suggestion7") + .field("weight", 5) + .startObject("contexts") + .array("loc1", "ezs42e44yx96") + .array("loc2", "wh0n9447fwrc") + .endObject() + .endObject() + .endArray() + .endObject(); + ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", builder.bytes()); + IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.names().indexName()); + assertContextSuggestFields(fields, 6); + } + @Test + public void testParsingQueryContextBasic() throws Exception { + XContentBuilder builder = jsonBuilder().value("ezs42e44yx96"); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("geo", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + GeoQueryContext queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("ezs42e44yx96")); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.neighbours.length, equalTo(1)); + } + + @Test + public void testParsingQueryContextGeoPoint() throws Exception { + XContentBuilder builder = jsonBuilder().startObject() + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("geo", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + GeoQueryContext queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("wh0n9447fwrc")); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.neighbours.length, equalTo(1)); + } + + @Test + public void testParsingQueryContextObject() throws Exception { + XContentBuilder builder = jsonBuilder().startObject() + .startObject("context") + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject() + .field("boost", 10) + .array("neighbours", 1, 2, 3) + .endObject(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("geo", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + GeoQueryContext queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("wh0n9447fwrc")); + assertThat(queryContext.boost, equalTo(10)); + assertThat(queryContext.neighbours.length, equalTo(3)); + } + + @Test + public void testParsingQueryContextObjectArray() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .startObject("context") + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject() + .field("boost", 10) + .array("neighbours", 1, 2, 3) + .endObject() + .startObject() + .startObject("context") + .field("lat", 22.337374) + .field("lon", 92.112583) + .endObject() + .field("boost", 2) + .array("neighbours", 3) + .endObject() + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("geo", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + GeoQueryContext queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("wh0n9447fwrc")); + assertThat(queryContext.boost, equalTo(10)); + assertThat(queryContext.neighbours.length, equalTo(3)); + assertTrue(iterator.hasNext()); + queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("w5cx046kdu24")); + assertThat(queryContext.boost, equalTo(2)); + assertThat(queryContext.neighbours.length, equalTo(1)); + } + + @Test + public void testParsingQueryContextMixed() throws Exception { + XContentBuilder builder = jsonBuilder().startArray() + .startObject() + .startObject("context") + .field("lat", 23.654242) + .field("lon", 90.047153) + .endObject() + .field("boost", 10) + .array("neighbours", 1, 2, 3) + .endObject() + .startObject() + .field("lat", 22.337374) + .field("lon", 92.112583) + .endObject() + .endArray(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.bytes()); + GeoContextMapping mapping = ContextBuilder.geo("geo").build(); + ContextMapping.QueryContexts queryContexts = mapping.parseQueryContext("geo", parser); + Iterator iterator = queryContexts.iterator(); + assertTrue(iterator.hasNext()); + GeoQueryContext queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("wh0n9447fwrc")); + assertThat(queryContext.boost, equalTo(10)); + assertThat(queryContext.neighbours.length, equalTo(3)); + assertTrue(iterator.hasNext()); + queryContext = iterator.next(); + assertThat(queryContext.geoHash.toString(), equalTo("w5cx046kdu24")); + assertThat(queryContext.boost, equalTo(1)); + assertThat(queryContext.neighbours.length, equalTo(1)); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProviderV1.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/old/AnalyzingCompletionLookupProviderV1.java similarity index 95% rename from core/src/test/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProviderV1.java rename to core/src/test/java/org/elasticsearch/search/suggest/completion/old/AnalyzingCompletionLookupProviderV1.java index 23f92bd7ed33f..d097e3dc60634 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProviderV1.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/old/AnalyzingCompletionLookupProviderV1.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import com.carrotsearch.hppc.ObjectLongHashMap; @@ -47,11 +47,12 @@ import org.apache.lucene.util.fst.PositiveIntOutputs; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; -import org.elasticsearch.search.suggest.completion.AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.CompletionLookupProvider; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextQuery; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.search.suggest.completion.old.AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder; +import org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat.CompletionLookupProvider; +import org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat.LookupFactory; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping.ContextQuery; import java.io.IOException; import java.util.Collection; @@ -234,7 +235,7 @@ public LookupFactory load(IndexInput input) throws IOException { final long ramBytesUsed = sizeInBytes; return new LookupFactory() { @Override - public Lookup getLookup(CompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext) { + public Lookup getLookup(OldCompletionFieldMapper.CompletionFieldType fieldType, CompletionSuggestionContext suggestionContext) { AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(fieldType.names().indexName()); if (analyzingSuggestHolder == null) { return null; diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionPostingsFormatTest.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/old/CompletionPostingsFormatTest.java similarity index 95% rename from core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionPostingsFormatTest.java rename to core/src/test/java/org/elasticsearch/search/suggest/completion/old/CompletionPostingsFormatTest.java index 1b29173973bae..5bd8ca8c59902 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionPostingsFormatTest.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/old/CompletionPostingsFormatTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.search.suggest.completion; +package org.elasticsearch.search.suggest.completion.old; import com.google.common.collect.Lists; @@ -55,10 +55,10 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType.Names; -import org.elasticsearch.index.mapper.core.CompletionFieldMapper; +import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper; import org.elasticsearch.search.suggest.SuggestUtils; -import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory; -import org.elasticsearch.search.suggest.context.ContextMapping; +import org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat.LookupFactory; +import org.elasticsearch.search.suggest.completion.old.context.ContextMapping; import org.elasticsearch.test.ESTestCase; import org.junit.Test; @@ -76,7 +76,7 @@ public class CompletionPostingsFormatTest extends ESTestCase { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - static final CompletionFieldMapper.CompletionFieldType FIELD_TYPE = CompletionFieldMapper.Defaults.FIELD_TYPE.clone(); + static final OldCompletionFieldMapper.CompletionFieldType FIELD_TYPE = OldCompletionFieldMapper.Defaults.FIELD_TYPE.clone(); static final NamedAnalyzer analyzer = new NamedAnalyzer("foo", new StandardAnalyzer()); static { FIELD_TYPE.setNames(new Names("foo")); @@ -97,7 +97,7 @@ public void testCompletionPostingsFormat() throws IOException { IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); LookupFactory load = currentProvider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); + OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(currentProvider); Lookup lookup = load.getLookup(fieldType, new CompletionSuggestionContext(null)); List result = lookup.lookup("ge", false, 10); @@ -116,7 +116,7 @@ public void testProviderBackwardCompatibilityForVersion1() throws IOException { IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); LookupFactory load = currentProvider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); + OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(currentProvider); AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder analyzingSuggestHolder = load.getAnalyzingSuggestHolder(fieldType); assertThat(analyzingSuggestHolder.sepLabel, is(AnalyzingCompletionLookupProviderV1.SEP_LABEL)); @@ -134,7 +134,7 @@ public void testProviderVersion2() throws IOException { IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); LookupFactory load = currentProvider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); + OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(currentProvider); AnalyzingCompletionLookupProvider.AnalyzingSuggestHolder analyzingSuggestHolder = load.getAnalyzingSuggestHolder(fieldType); assertThat(analyzingSuggestHolder.sepLabel, is(XAnalyzingSuggester.SEP_LABEL)); @@ -243,9 +243,9 @@ public boolean hasContexts() { reference.build(iter); AnalyzingCompletionLookupProvider currentProvider = new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, usePayloads); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); + OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(currentProvider); - final CompletionFieldMapper mapper = new CompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE, indexSettings, FieldMapper.MultiFields.empty(), null); + final OldCompletionFieldMapper mapper = new OldCompletionFieldMapper("foo", fieldType, Integer.MAX_VALUE, indexSettings, FieldMapper.MultiFields.empty(), null); Lookup buildAnalyzingLookup = buildAnalyzingLookup(mapper, titles, titles, weights); Field field = buildAnalyzingLookup.getClass().getDeclaredField("maxAnalyzedPathsForOneInput"); field.setAccessible(true); @@ -281,7 +281,7 @@ public void nextToken() throws IOException { } } - public Lookup buildAnalyzingLookup(final CompletionFieldMapper mapper, String[] terms, String[] surfaces, long[] weights) + public Lookup buildAnalyzingLookup(final OldCompletionFieldMapper mapper, String[] terms, String[] surfaces, long[] weights) throws IOException { RAMDirectory dir = new RAMDirectory(); Codec codec = new Lucene50Codec() { @@ -344,7 +344,7 @@ public int size() { IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); LookupFactory load = provider.load(input); - CompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); + OldCompletionFieldMapper.CompletionFieldType fieldType = FIELD_TYPE.clone(); fieldType.setProvider(provider); assertNull(load.getLookup(fieldType, new CompletionSuggestionContext(null))); dir.close(); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/context/GeoLocationContextMappingTest.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/old/context/GeoLocationContextMappingTest.java similarity index 98% rename from core/src/test/java/org/elasticsearch/search/suggest/context/GeoLocationContextMappingTest.java rename to core/src/test/java/org/elasticsearch/search/suggest/completion/old/context/GeoLocationContextMappingTest.java index f1c1761855628..793acc916105a 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/context/GeoLocationContextMappingTest.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/old/context/GeoLocationContextMappingTest.java @@ -16,14 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.search.suggest.context; +package org.elasticsearch.search.suggest.completion.old.context; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.suggest.context.ContextMapping.ContextConfig; import org.elasticsearch.test.ESTestCase; import org.junit.Test; @@ -32,6 +31,7 @@ import java.util.HashMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.search.suggest.completion.old.context.ContextMapping.*; /** * diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index 456dc8ba94019..dae4281e971e9 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -4,45 +4,37 @@ NOTE: In order to understand the format of suggestions, please read the <> page first. -The `completion` suggester is a so-called prefix suggester. It does not -do spell correction like the `term` or `phrase` suggesters but allows -basic `auto-complete` functionality. - -==== Why another suggester? Why not prefix queries? - -The first question which comes to mind when reading about a prefix -suggestion is, why you should use it at all, if you have prefix queries -already. The answer is simple: Prefix suggestions are fast. - -The data structures are internally backed by Lucenes -`AnalyzingSuggester`, which uses FSTs to execute suggestions. Usually -these data structures are costly to create, stored in-memory and need to -be rebuilt every now and then to reflect changes in your indexed -documents. The `completion` suggester circumvents this by storing the -FST as part of your index during index time. This allows for really fast -loads and executions. +The `completion` suggester provides auto-complete/search-as-you-type +functionality. This is a navigational feature to guide users to +relevant results as they are typing, improving search precision. +It is not meant for spell correction or did-you-mean functionality +like the `term` or `phrase` suggesters. + +Ideally, auto-complete functionality should be as fast as a user +types to provide instant feedback relevant to what a user has already +typed in. Hence, `completion` suggester is optimized for speed. +The suggester uses data structures that enable fast lookups, +but are costly to build and are stored in-memory. [[completion-suggester-mapping]] ==== Mapping -In order to use this feature, you have to specify a special mapping for -this field, which enables the special storage of the field. +To use this feature, specify a special mapping for this field, +which indexes the field values for fast completions. [source,js] -------------------------------------------------- -curl -X PUT localhost:9200/music -curl -X PUT localhost:9200/music/song/_mapping -d '{ +PUT music/song/_mapping +{ "song" : { "properties" : { - "name" : { "type" : "string" }, - "suggest" : { "type" : "completion", - "analyzer" : "simple", - "search_analyzer" : "simple", - "payloads" : true + ... + "suggest" : { + "type" : "completion", } } } -}' +} -------------------------------------------------- Mapping supports the following parameters: @@ -57,17 +49,14 @@ Mapping supports the following parameters: `search_analyzer`:: The search analyzer to use, defaults to value of `analyzer`. -`payloads`:: - Enables the storing of payloads, defaults to `false` - `preserve_separators`:: Preserves the separators, defaults to `true`. If disabled, you could find a field starting with `Foo Fighters`, if you suggest for `foof`. `preserve_position_increments`:: - Enables position increments, defaults - to `true`. If disabled and using stopwords analyzer, you could get a + Enables position increments, defaults to `true`. + If disabled and using stopwords analyzer, you could get a field starting with `The Beatles`, if you suggest for `b`. *Note*: You could also achieve this by indexing two inputs, `Beatles` and `The Beatles`, no need to change a simple analyzer, if you are able to @@ -77,24 +66,22 @@ Mapping supports the following parameters: Limits the length of a single input, defaults to `50` UTF-16 code points. This limit is only used at index time to reduce the total number of characters per input string in order to prevent massive inputs from - bloating the underlying datastructure. The most usecases won't be influenced - by the default value since prefix completions hardly grow beyond prefixes longer - than a handful of characters. (Old name "max_input_len" is deprecated) + bloating the underlying datastructure. Most usecases won't be influenced + by the default value since prefix completions seldom grow beyond prefixes longer + than a handful of characters. [[indexing]] ==== Indexing [source,js] -------------------------------------------------- -curl -X PUT 'localhost:9200/music/song/1?refresh=true' -d '{ - "name" : "Nevermind", +PUT music/song/1?refresh=true +{ "suggest" : { "input": [ "Nevermind", "Nirvana" ], - "output": "Nirvana - Nevermind", - "payload" : { "artistId" : 2321 }, "weight" : 34 } -}' +} -------------------------------------------------- The following parameters are supported: @@ -103,42 +90,21 @@ The following parameters are supported: The input to store, this can be a an array of strings or just a string. This field is mandatory. -`output`:: - The string to return, if a suggestion matches. This is very - useful to normalize outputs (i.e. have them always in the format - `artist - songname`). This is optional. - *Note*: The result is de-duplicated if several documents - have the same output, i.e. only one is returned as part of the - suggest result. - -`payload`:: - An arbitrary JSON object, which is simply returned in the - suggest option. You could store data like the id of a document, in order - to load it from elasticsearch without executing another search (which - might not yield any results, if `input` and `output` differ strongly). - `weight`:: A positive integer or a string containing a positive integer, which defines a weight and allows you to rank your suggestions. This field is optional. -NOTE: Even though you will lose most of the features of the -completion suggest, you can choose to use the following shorthand form. -Keep in mind that you will not be able to use several inputs, an output, -payloads or weights. This form does still work inside of multi fields. +You can use the following shorthand form. Note that you can not specify +a weight with suggestion(s). [source,js] -------------------------------------------------- { - "suggest" : "Nirvana" + "suggest" : [ "Nevermind", "Nirvana" ] } -------------------------------------------------- -NOTE: The suggest data structure might not reflect deletes on -documents immediately. You may need to do an <> for that. -You can call optimize with the `only_expunge_deletes=true` to only cater for deletes -or alternatively call a <> operation. - [[querying]] ==== Querying @@ -147,14 +113,15 @@ type as `completion`. [source,js] -------------------------------------------------- -curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ +POST music/_suggest?pretty +{ "song-suggest" : { - "text" : "n", + "prefix" : "n", "completion" : { "field" : "suggest" } } -}' +} { "_shards" : { @@ -167,18 +134,15 @@ curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ "offset" : 0, "length" : 4, "options" : [ { - "text" : "Nirvana - Nevermind", - "score" : 34.0, "payload" : {"artistId":2321} + "text" : "Nirvana", + "score" : 34.0 } ] } ] } -------------------------------------------------- -As you can see, the payload is included in the response, if configured -appropriately. If you configured a weight for a suggestion, this weight -is used as `score`. Also the `text` field uses the `output` of your -indexed suggestion, if configured, otherwise the matched part of the -`input` field. +The configured weight for a suggestion is returned as `score`. +The `text` field uses the `input` of your indexed suggestion. The basic completion suggester query supports the following two parameters: @@ -189,17 +153,21 @@ NOTE: The completion suggester considers all documents in the index. See <> for an explanation of how to query a subset of documents instead. +NOTE: It will be possible to return the associated documents +with each suggestion in the future (TODO). + [[fuzzy]] ==== Fuzzy queries The completion suggester also supports fuzzy queries - this means, -you can actually have a typo in your search and still get results back. +you can have a typo in your search and still get results back. [source,js] -------------------------------------------------- -curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ +POST music/_suggest?pretty +{ "song-suggest" : { - "text" : "n", + "prefix" : "n", "completion" : { "field" : "suggest", "fuzzy" : { @@ -207,9 +175,12 @@ curl -X POST 'localhost:9200/music/_suggest?pretty' -d '{ } } } -}' +} -------------------------------------------------- +Suggestions that share the longest prefix to the query `prefix` will +be scored higher. + The fuzzy query can take specific fuzzy parameters. The following parameters are supported: @@ -231,10 +202,48 @@ The following parameters are supported: checked for fuzzy alternatives, defaults to `1` `unicode_aware`:: - Sets all are measurements (like edit distance, - transpositions and lengths) in unicode code points - (actual letters) instead of bytes. + If `true`, all measurements (like fuzzy edit + distance, transpositions, and lengths) are + measured in Unicode code points instead of + in bytes. This is slightly slower than raw + bytes, so it is set to `false` by default. NOTE: If you want to stick with the default values, but still use fuzzy, you can either use `fuzzy: {}` or `fuzzy: true`. + +[[regex]] +==== Regex queries + +The completion suggester also supports regex queries meaning +you can express a prefix as a regular expression + +[source,js] +-------------------------------------------------- +POST music/_suggest?pretty +{ + "song-suggest" : { + "regex" : "n[ever|i]r", + "completion" : { + "field" : "suggest", + } + } +} +-------------------------------------------------- + +The regex query can take specific regex parameters. +The following parameters are supported: + +[horizontal] +`flags`:: + Possible flags are `ALL` (default), `ANYSTRING`, `COMPLEMENT`, + `EMPTY`, `INTERSECTION`, `INTERVAL`, or `NONE`. See <> + for their meaning + +`max_determinized_states`:: + Regular expressions are dangerous because it's easy to accidentally + create an innocuous looking one that requires an exponential number of + internal determinized automaton states (and corresponding RAM and CPU) + for Lucene to execute. Lucene prevents these using the + `max_determinized_states` setting (defaults to 10000). You can raise + this limit to allow more complex regular expressions to execute. diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc index c09659a43a94a..9711776cfc414 100644 --- a/docs/reference/search/suggesters/context-suggest.asciidoc +++ b/docs/reference/search/suggesters/context-suggest.asciidoc @@ -1,237 +1,290 @@ [[suggester-context]] === Context Suggester -The context suggester is an extension to the suggest API of Elasticsearch. Namely the -suggester system provides a very fast way of searching documents by handling these -entirely in memory. But this special treatment does not allow the handling of -traditional queries and filters, because those would have notable impact on the -performance. So the context extension is designed to take so-called context information -into account to specify a more accurate way of searching within the suggester system. -Instead of using the traditional query and filter system a predefined ``context`` is -configured to limit suggestions to a particular subset of suggestions. -Such a context is defined by a set of context mappings which can either be a simple -*category* or a *geo location*. The information used by the context suggester is -configured in the type mapping with the `context` parameter, which lists all of the -contexts that need to be specified in each document and in each suggestion request. -For instance: +The completion suggester considers all documents in the index, but it is often +desirable to serve suggestions filtered and/or boosted by some criteria. +For example, you want to suggest song titles filtered by certain artists or +you want to boost song titles based on their genre. + +To achieve suggestion filtering and/or boosting, you can add context mappings while +configuring a completion field. You can define multiple context mappings for a +completion field. +Every context mapping has a unique name and a type. There are two types: `category` +and `geo`. Context mappings are configured under the `contexts` parameter in +the field mapping. + +The following defines two context mappings for a completion field: [source,js] -------------------------------------------------- -PUT services/_mapping/service +PUT place/shops/_mapping { - "service": { - "properties": { - "name": { - "type" : "string" - }, - "tag": { - "type" : "string" - }, - "suggest_field": { - "type": "completion", - "context": { - "color": { <1> + "shops" : { + "properties" : { + ... + "suggest" : { + "type" : "completion", + "contexts": [ + { <1> + "name": "place_type", "type": "category", - "path": "color_field", - "default": ["red", "green", "blue"] + "path": "cat" }, - "location": { <2> + { <2> + "name": "location" "type": "geo", - "precision": "5m", - "neighbors": true, - "default": "u33" + "path": "loc" } - } + ] } } } } -------------------------------------------------- -<1> See <> -<2> See <> +<1> Defines a `category` context named 'place_type', which will index values from field 'cat'. + See <> +<2> Defines a `geo` context named 'location', which will index values from field 'loc'. + See <> -However contexts are specified (as type `category` or `geo`, which are discussed below), each -context value generates a new sub-set of documents which can be queried by the completion -suggester. All three types accept a `default` parameter which provides a default value to use -if the corresponding context value is absent. - -The basic structure of this element is that each field forms a new context and the fieldname -is used to reference this context information later on during indexing or querying. All context -mappings have the `default` and the `type` option in common. The value of the `default` field -is used, when ever no specific is provided for the certain context. Note that a context is -defined by at least one value. The `type` option defines the kind of information hold by this -context. These type will be explained further in the following sections. +NOTE: Adding context mappings increases the index size for completion field. The completion index +is entirely heap resident, you can monitor the completion field index size using <>. [[suggester-context-category]] [float] ==== Category Context -The `category` context allows you to specify one or more categories in the document at index time. -The document will be assigned to each named category, which can then be queried later. The category -type also allows to specify a field to extract the categories from. The `path` parameter is used to -specify this field of the documents that should be used. If the referenced field contains multiple -values, all these values will be used as alternative categories. + +The `category` context allows you to associate one or more categories with suggestions at index +time. At query time, suggestions can be filtered and boosted by their associated categories. [float] ===== Category Mapping -The mapping for a category is simply defined by its `default` values. These can either be -defined as list of *default* categories: +A `category` context mapping, where categories are provided explicitly with suggestions +can be defined as follows: [source,js] -------------------------------------------------- -"context": { - "color": { +"contexts": [ + { + "name": "cat_context", "type": "category", - "default": ["red", "orange"] } -} +] -------------------------------------------------- -or as a single value +Alternatively, A `category` context mapping that references another field within a document +can be defined as follows: [source,js] -------------------------------------------------- -"context": { - "color": { +"contexts": [ + { + "name": "cat_context", "type": "category", - "default": "red" + "path": "cat_field" + } +] +-------------------------------------------------- + +[float] +===== Indexing category contexts + +Category contexts can be specified explicitly when indexing suggestions. If a suggestion has +multiple categories, the suggestion will be indexed for each category: + +[source,js] +-------------------------------------------------- +PUT place/shops/1 +{ + ... + "suggest": { + "input": ["timmy's", "starbucks", "dunkin donuts"], + "context": { + "place_type": ["cafe", "food"] <1> + } } } -------------------------------------------------- -or as reference to another field within the documents indexed: +<1> These suggestions will be associated with 'cafe' and 'food' category. + +Category contexts can also be referenced from another indexed field in the document via +the `path` parameter in the field mapping: [source,js] -------------------------------------------------- -"context": { - "color": { +"contexts": [ + { + "name": "cat_context", "type": "category", - "default": "red", - "path": "color_field" + "path": "cat" } +] +-------------------------------------------------- + +With the above mapping, the following will index the suggestions, treating the values of the +'cat' field as category contexts: + +[source,js] +-------------------------------------------------- +PUT place/shops/1 +{ + ... + "suggest": ["timmy's", "starbucks", "dunkin donuts"], + "cat": ["cafe", "food"] <1> } -------------------------------------------------- -in this case the *default* categories will only be used, if the given field does not -exist within the document. In the example above the categories are received from a -field named `color_field`. If this field does not exist a category *red* is assumed for -the context *color*. +<1> These suggestions will be associated with 'cafe' and 'food' category. + +NOTE: If context mapping references another field and the categories +are explicitly indexed, the suggestions are indexed with both set +of categories. + [float] -===== Indexing category contexts -Within a document the category is specified either as an `array` of values, a -single value or `null`. A list of values is interpreted as alternative categories. So -a document belongs to all the categories defined. If the category is `null` or remains -unset the categories will be retrieved from the documents field addressed by the `path` -parameter. If this value is not set or the field is missing, the default values of the -mapping will be assigned to the context. +===== Category Query + +Suggestions can be filtered by one or more categories. The following +filters suggestions by multiple categories: [source,js] -------------------------------------------------- -PUT services/service/1 +POST place/_suggest?pretty { - "name": "knapsack", - "suggest_field": { - "input": ["knacksack", "backpack", "daypack"], - "context": { - "color": ["red", "yellow"] + "suggest" : { + "prefix" : "tim", + "completion" : { + "field" : "suggest", + "size": 10, + "contexts": { + "place_type": [ "cafe", "restaurants" ] + } } } } -------------------------------------------------- -[float] -===== Category Query -A query within a category works similar to the configuration. If the value is `null` -the mappings default categories will be used. Otherwise the suggestion takes place -for all documents that have at least one category in common with the query. +NOTE: When no categories are provided at query-time, all indexed documents are considered. +Querying with no categories on a category enabled completion field should be avoided, as it +will degrade search performance. + +Suggestions with certain categories can be boosted higher than others. +The following filters suggestions by categories and additionally boosts +suggestions associated with some categories: [source,js] -------------------------------------------------- -POST services/_suggest?pretty' +POST place/_suggest?pretty { "suggest" : { - "text" : "m", + "prefix" : "tim", "completion" : { - "field" : "suggest_field", + "field" : "suggest", "size": 10, - "context": { - "color": "red" + "contexts": { + "place_type": [ <1> + { "context" : "cafe" }, + { "context" : "restaurants", "boost": 2 } + ] } } } } -------------------------------------------------- +<1> The context query filter suggestions associated with + categories 'cafe' and 'restaurants' and boosts the + suggestions associated with 'restaurants' by a + factor of `2` + +In addition to accepting category values, a context query can be composed of +multiple category context clauses. The following parameters are supported for a +`category` context clause: + +[horizontal] +`context`:: + The value of the category to filter/boost on. + This is mandatory. + +`boost`:: + The factor by which the score of the suggestion + should be boosted, the score is computed by + multiplying the boost with the suggestion weight, + defaults to `1` + +`prefix`:: + Whether the category value should be treated as a + prefix or not. For example, if set to `true`, + you can filter category of 'type1', 'type2' and + so on, by specifying a category prefix of 'type'. + Defaults to `false` [[suggester-context-geo]] [float] ==== Geo location Context -A `geo` context allows you to limit results to those that lie within a certain distance -of a specified geolocation. At index time, a lat/long geo point is converted into a -geohash of a certain precision, which provides the context. + +A `geo` context allows you to associate one or more geo points or geohashes with suggestions +at index time. At query time, suggestions can be filtered and boosted if they are within +a certain distance of a specified geo location. + +Internally, geo points are encoded as geohashes with specified precision. See <> for +more details. [float] -===== Geo location Mapping -The mapping for a geo context accepts four settings, only of which `precision` is required: +===== Geo Mapping + +In addition to the `path` setting, `geo` context mapping accepts the following settings: [horizontal] -`precision`:: This defines the precision of the geohash and can be specified as `5m`, `10km`, - or as a raw geohash precision: `1`..`12`. It's also possible to setup multiple - precisions by defining a list of precisions: `["5m", "10km"]` -`neighbors`:: Geohashes are rectangles, so a geolocation, which in reality is only 1 metre - away from the specified point, may fall into the neighbouring rectangle. Set - `neighbours` to `true` to include the neighbouring geohashes in the context. - (default is *on*) -`path`:: Optionally specify a field to use to look up the geopoint. -`default`:: The geopoint to use if no geopoint has been specified. - -Since all locations of this mapping are translated into geohashes, each location matches -a geohash cell. So some results that lie within the specified range but not in the same -cell as the query location will not match. To avoid this the `neighbors` option allows a -matching of cells that join the bordering regions of the documents location. This option -is turned on by default. -If a document or a query doesn't define a location a value to use instead can defined by -the `default` option. The value of this option supports all the ways a `geo_point` can be -defined. The `path` refers to another field within the document to retrieve the -location. If this field contains multiple values, the document will be linked to all these -locations. +`precision`:: + This defines the precision of the geohash to be indexed and can be specified + as a distance value (`5m`, `10km` etc.), or as a raw geohash precision (`1`..`12`). + Defaults to a raw geohash precision value of `6`. + +NOTE: The index time `precision` setting sets the maximum geohash precision that +can be used at query time. + +The following defines a `geo` context mapping with an index time precision of `4` +indexing values from a geo point field 'pin': [source,js] -------------------------------------------------- -"context": { - "location": { +"contexts": [ + { + "name": "location" "type": "geo", - "precision": ["1km", "5m"], - "neighbors": true, + "precision": 4, "path": "pin", - "default": { - "lat": 0.0, - "lon": 0.0 - } } -} +] -------------------------------------------------- [float] -===== Geo location Config +===== Indexing geo contexts -Within a document a geo location retrieved from the mapping definition can be overridden -by another location. In this case the context mapped to a geo location supports all -variants of defining a `geo_point`. +`geo` contexts can be explicitly set with suggestions or be indexed from a geo point field in the +document via the `path` parameter, similar to `category` contexts. Associating multiple geo location context +with a suggestion, will index the suggestion for every geo location. The following indexes a suggestion +with two geo location contexts: [source,js] -------------------------------------------------- -PUT services/service/1 +PUT place/shops/1 { - "name": "some hotel 1", - "suggest_field": { - "input": ["my hotel", "this hotel"], - "context": { - "location": { - "lat": 0, - "lon": 0 - } - } + "suggest": { + "input": "timmy's", + "context": [ + "location": [ + { + "lat": 43.6624803, + "lon": -79.3863353 + }, + { + "lat": 43.6624718, + "lon": -79.3873227 + } + ] + ] } } -------------------------------------------------- @@ -239,24 +292,23 @@ PUT services/service/1 [float] ===== Geo location Query -Like in the configuration, querying with a geo location in context, the geo location -query supports all representations of a `geo_point` to define the location. In this -simple case all precision values defined in the mapping will be applied to the given -location. +Suggestions can be filtered and boosted with respect to how close they are to one or +more geo points. The following filters suggestions that fall within the area represented by +the encoded geohash of a geo point: [source,js] -------------------------------------------------- -POST services/_suggest +POST place/_suggest { "suggest" : { - "text" : "m", + "prefix" : "tim", "completion" : { - "field" : "suggest_field", + "field" : "suggest", "size": 10, "context": { "location": { - "lat": 0, - "lon": 0 + "lat": 43.662, + "lon": -79.380 } } } @@ -264,54 +316,71 @@ POST services/_suggest } -------------------------------------------------- -But it also possible to set a subset of the precisions set in the mapping, by using the -`precision` parameter. Like in the mapping, this parameter is allowed to be set to a -single precision value or a list of these. +NOTE: When a location with a lower precision at query time is specified, all suggestions +that fall within the area will be considered. + +Suggestions that are within an area represented by a geohash can also be boosted higher +than others, as shown by the following: [source,js] -------------------------------------------------- -POST services/_suggest +POST place/_suggest?pretty { "suggest" : { - "text" : "m", + "prefix" : "tim", "completion" : { - "field" : "suggest_field", + "field" : "suggest", "size": 10, - "context": { - "location": { - "value": { - "lat": 0, - "lon": 0 + "contexts": { + "location": [ <1> + { + "lat": 43.6624803, + "lon": -79.3863353, + "precision": 2 }, - "precision": "1km" - } + { + "context": { + "lat": 43.6624803, + "lon": -79.3863353 + }, + "boost": 2 + } + ] } } } } -------------------------------------------------- +<1> The context query filters for suggestions that fall under + the geo location represented by a geohash of '(43.662, -79.380)' + with a precision of '2' and boosts suggestions + that fall under the geohash representation of '(43.6624803, -79.3863353)' + with a default precision of '6' by a factor of `2` -A special form of the query is defined by an extension of the object representation of -the `geo_point`. Using this representation allows to set the `precision` parameter within -the location itself: +In addition to accepting context values, a context query can be composed of +multiple context clauses. The following parameters are supported for a +`category` context clause: -[source,js] --------------------------------------------------- -POST services/_suggest -{ - "suggest" : { - "text" : "m", - "completion" : { - "field" : "suggest_field", - "size": 10, - "context": { - "location": { - "lat": 0, - "lon": 0, - "precision": "1km" - } - } - } - } -} --------------------------------------------------- +[horizontal] +`context`:: + A geo point object or a geo hash string to filter or + boost the suggestion by. This is mandatory. + +`boost`:: + The factor by which the score of the suggestion + should be boosted, the score is computed by + multiplying the boost with the suggestion weight, + defaults to `1` + +`precision`:: + The precision of the geohash to encode the query geo point. + This can be specified as a distance value (`5m`, `10km` etc.), + or as a raw geohash precision (`1`..`12`). + Defaults to index time precision level. + +`neighbours`:: + Accepts an array of precision values at which + neighbouring geohashes should be taken into account. + precision value can be a distance value (`5m`, `10km` etc.) + or a raw geohash precision (`1`..`12`). Defaults to + generating neighbours for index time precision level. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml index e9af003367735..b14e4cb488555 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/13_fields.yaml @@ -35,6 +35,27 @@ setup: - do: indices.refresh: {} + - do: + suggest: + index: test1 + body: + result: + text: "b" + completion: + field: bar.completion + + - do: + suggest: + index: test1 + body: + result: + text: "b" + completion: + field: baz.completion + + - do: + indices.refresh: {} + - do: search: sort: bar,baz diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml new file mode 100644 index 0000000000000..67a9980a2aa56 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml @@ -0,0 +1,229 @@ +# This test creates one huge mapping in the setup +# Every test should use its own field to make sure it works + +setup: + + - do: + indices.create: + index: test + body: + mappings: + test: + "properties": + "suggest_1": + "type" : "completion" + "suggest_2": + "type" : "completion" + "suggest_3": + "type" : "completion" + "suggest_4": + "type" : "completion" + "suggest_5a": + "type" : "completion" + "suggest_5b": + "type" : "completion" + +--- +"Simple suggestion should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_1: "bar" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_1: "baz" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_1 + + - length: { result: 1 } + - length: { result.0.options: 2 } + +--- +"Simple suggestion array should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_2: ["bar", "foo"] + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "f" + completion: + field: suggest_2 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "foo" } + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_2 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "bar" } + +--- +"Suggestion entry should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_3: + input: "bar" + weight: 2 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_3: + input: "baz" + weight: 3 + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_3 + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "baz" } + - match: { result.0.options.1.text: "bar" } + +--- +"Suggestion entry array should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_4: + - input: "bar" + weight: 3 + - input: "fo" + weight: 3 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_4: + - input: "baz" + weight: 2 + - input: "foo" + weight: 1 + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_4 + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "bar" } + - match: { result.0.options.1.text: "baz" } + + - do: + suggest: + body: + result: + text: "f" + completion: + field: suggest_4 + + - length: { result: 1 } + - length: { result.0.options: 2 } + - match: { result.0.options.0.text: "fo" } + - match: { result.0.options.1.text: "foo" } + +--- +"Multiple Completion fields should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_5a: "bar" + suggest_5b: "baz" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_5a + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "bar" } + + - do: + suggest: + body: + result: + text: "b" + completion: + field: suggest_5b + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "baz" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_context.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_context.yaml deleted file mode 100644 index cabd8a39552f0..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_context.yaml +++ /dev/null @@ -1,224 +0,0 @@ -# This test creates one huge mapping in the setup -# Every test should use its own field to make sure it works - -setup: - - - do: - indices.create: - index: test - body: - mappings: - test: - "properties": - "suggest_context": - "type" : "completion" - "context": - "color": - "type" : "category" - "suggest_context_default_hardcoded": - "type" : "completion" - "context": - "color": - "type" : "category" - "default" : "red" - "suggest_context_default_path": - "type" : "completion" - "context": - "color": - "type" : "category" - "default" : "red" - "path" : "color" - "suggest_geo": - "type" : "completion" - "context": - "location": - "type" : "geo" - "precision" : "5km" - ---- -"Simple context suggestion should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_context: - input: "Hoodie red" - context: - color: "red" - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_context: - input: "Hoodie blue" - context: - color: "blue" - - - do: - indices.refresh: {} - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context - context: - color: "red" - - - match: {result.0.options.0.text: "Hoodie red" } - ---- -"Hardcoded category value should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_context_default_hardcoded: - input: "Hoodie red" - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_context_default_hardcoded: - input: "Hoodie blue" - context: - color: "blue" - - - do: - indices.refresh: {} - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context_default_hardcoded - context: - color: "red" - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hoodie red" } - - ---- -"Category suggest context default path should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_context_default_path: - input: "Hoodie red" - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_context_default_path: - input: "Hoodie blue" - color: "blue" - - - do: - indices.refresh: {} - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context_default_path - context: - color: "red" - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hoodie red" } - - - do: - suggest: - body: - result: - text: "hoo" - completion: - field: suggest_context_default_path - context: - color: "blue" - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hoodie blue" } - - ---- -"Geo suggest should work": - - - do: - index: - index: test - type: test - id: 1 - body: - suggest_geo: - input: "Hotel Marriot in Amsterdam" - context: - location: - lat : 52.22 - lon : 4.53 - - - do: - index: - index: test - type: test - id: 2 - body: - suggest_geo: - input: "Hotel Marriot in Berlin" - context: - location: - lat : 53.31 - lon : 13.24 - - - do: - indices.refresh: {} - - - do: - indices.get_mapping: {} - - - do: - suggest: - index: test - body: - result: - text: "hote" - completion: - field: suggest_geo - context: - location: - lat : 52.22 - lon : 4.53 - - - length: { result: 1 } - - length: { result.0.options: 1 } - - match: { result.0.options.0.text: "Hotel Marriot in Amsterdam" } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml new file mode 100644 index 0000000000000..c16e994649dd0 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/suggest/30_context.yaml @@ -0,0 +1,267 @@ +# This test creates one huge mapping in the setup +# Every test should use its own field to make sure it works + +setup: + + - do: + indices.create: + index: test + body: + mappings: + test: + "properties": + "location": + "type": "geo_point" + "suggest_context": + "type" : "completion" + "contexts": + - "name" : "color" + "type" : "category" + "suggest_context_with_path": + "type" : "completion" + "contexts": + - "name" : "color" + "type" : "category" + "path" : "color" + "suggest_geo": + "type" : "completion" + "contexts": + - "name" : "location" + "type" : "geo" + "precision" : "5km" + "suggest_multi_contexts": + "type" : "completion" + "contexts": + - "name" : "location" + "type" : "geo" + "precision" : "5km" + "path" : "location" + - "name" : "color" + "type" : "category" + "path" : "color" + +--- +"Simple context suggestion should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_context: + input: "foo red" + contexts: + color: "red" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_context: + input: "foo blue" + contexts: + color: "blue" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context + contexts: + color: "red" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "foo red" } + +--- +"Category suggest context from path should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_context_with_path: + input: "Foo red" + contexts: + color: "red" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_context_with_path: "Foo blue" + color: "blue" + + - do: + indices.refresh: {} + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context_with_path + contexts: + color: "red" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Foo red" } + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context_with_path + contexts: + color: "blue" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Foo blue" } + + - do: + suggest: + body: + result: + text: "foo" + completion: + field: suggest_context_with_path + contexts: + color: ["blue", "red"] + + - length: { result: 1 } + - length: { result.0.options: 2 } + +--- +"Geo suggest should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_geo: + input: "Marriot in Amsterdam" + contexts: + location: + lat : 52.22 + lon : 4.53 + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_geo: + input: "Marriot in Berlin" + contexts: + location: + lat : 53.31 + lon : 13.24 + + - do: + indices.refresh: {} + + - do: + indices.get_mapping: {} + + - do: + suggest: + index: test + body: + result: + text: "mar" + completion: + field: suggest_geo + contexts: + location: + lat : 52.22 + lon : 4.53 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Marriot in Amsterdam" } + +--- +"Multi contexts should work": + + - do: + index: + index: test + type: test + id: 1 + body: + suggest_multi_contexts: "Marriot in Amsterdam" + location: + lat : 52.22 + lon : 4.53 + color: "red" + + - do: + index: + index: test + type: test + id: 2 + body: + suggest_multi_contexts: "Marriot in Berlin" + location: + lat : 53.31 + lon : 13.24 + color: "blue" + + - do: + indices.refresh: {} + + - do: + indices.get_mapping: {} + + - do: + suggest: + index: test + body: + result: + text: "mar" + completion: + field: suggest_multi_contexts + contexts: + location: + lat : 52.22 + lon : 4.53 + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Marriot in Amsterdam" } + + - do: + suggest: + index: test + body: + result: + text: "mar" + completion: + field: suggest_multi_contexts + contexts: + color: "blue" + + - length: { result: 1 } + - length: { result.0.options: 1 } + - match: { result.0.options.0.text: "Marriot in Berlin" }