Mask wildcard query special characters on keyword queries

server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java

@@ -79,6 +79,10 @@ private static String valueToString(Object value) {
     public final Query termQuery(Object value, QueryShardContext context) {
         String pattern = valueToString(value);
         if (matches(pattern, context)) {
+            if (context != null && context.getMapperService().hasNested()) {
+                // type filters are expected not to match nested docs
+                return Queries.newNonNestedFilter();
+            }
             return Queries.newMatchAllQuery();
         } else {
             return new MatchNoDocsQuery();

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -204,6 +204,7 @@ protected KeywordFieldType(KeywordFieldType ref) {
             this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
         }
 
+        @Override
         public KeywordFieldType clone() {
             return new KeywordFieldType(this);
         }

server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java

@@ -21,8 +21,6 @@
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
@@ -31,13 +29,16 @@
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 
@@ -47,6 +48,8 @@
  * can be implemented. */
 public abstract class StringFieldType extends TermBasedFieldType {
 
+    private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)");
+
     public StringFieldType() {}
 
     protected StringFieldType(MappedFieldType ref) {
@@ -92,16 +95,41 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
 
     @Override
     public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        Query termQuery = termQuery(value, context);
-        if (termQuery instanceof MatchNoDocsQuery || termQuery instanceof MatchAllDocsQuery) {
-            return termQuery;
-        }
-
+        failIfNotIndexed();
         if (context.allowExpensiveQueries() == false) {
             throw new ElasticsearchException("[wildcard] queries cannot be executed when '" +
                     ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false.");
         }
-        Term term = MappedFieldType.extractTerm(termQuery);
+
+        Term term;
+        if (searchAnalyzer() != null) {
+            // we want to normalize everything except wildcard characters, e.g. F?o Ba* to f?o ba*, even if e.g there
+            // is a char_filter that would otherwise remove them
+            Matcher wildcardMatcher = WILDCARD_PATTERN.matcher(value);
+            BytesRefBuilder sb = new BytesRefBuilder();
+            int last = 0;
+
+            while (wildcardMatcher.find()) {
+                if (wildcardMatcher.start() > 0) {
+                    String chunk = value.substring(last, wildcardMatcher.start());
+
+                    BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
+                    sb.append(normalized);
+                }
+                // append the matched group - without normalizing
+                sb.append(new BytesRef(wildcardMatcher.group()));
+
+                last = wildcardMatcher.end();
+            }
+            if (last < value.length()) {
+                String chunk = value.substring(last);
+                BytesRef normalized = searchAnalyzer().normalize(name(), chunk);
+                sb.append(normalized);
+            }
+            term = new Term(name(), sb.toBytesRef());
+        } else {
+            term = new Term(name(), indexedValueForSearch(value));
+        }
 
         WildcardQuery query = new WildcardQuery(term);
         QueryParsers.setRewriteMethod(query, method);

server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java

@@ -30,13 +30,12 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -52,6 +51,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
 public class TypeFieldMapper extends MetadataFieldMapper {
 
@@ -90,7 +90,7 @@ public MetadataFieldMapper getDefault(ParserContext context) {
         }
     }
 
-    public static final class TypeFieldType extends StringFieldType {
+    public static final class TypeFieldType extends ConstantFieldType {
 
         TypeFieldType() {
         }
@@ -121,61 +121,12 @@ public ValuesSourceType getValuesSourceType() {
         }
 
         @Override
-        public boolean isSearchable() {
-            return true;
-        }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new MatchAllDocsQuery();
-        }
-
-        @Override
-        public Query termQuery(Object value, QueryShardContext context) {
-            return termsQuery(Arrays.asList(value), context);
-        }
-
-        @Override
-        public Query termsQuery(List<?> values, QueryShardContext context) {
-            DocumentMapper mapper = context.getMapperService().documentMapper();
-            if (mapper == null) {
-                return new MatchNoDocsQuery("No types");
-            }
-            BytesRef indexType = indexedValueForSearch(mapper.type());
-            if (values.stream()
-                    .map(this::indexedValueForSearch)
-                    .anyMatch(indexType::equals)) {
-                if (context.getMapperService().hasNested()) {
-                    // type filters are expected not to match nested docs
-                    return Queries.newNonNestedFilter();
-                } else {
-                    return new MatchAllDocsQuery();
-                }
-            } else {
-                return new MatchNoDocsQuery("Type list does not contain the index type");
-            }
-        }
-
-        @Override
-        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
-            Query result = new MatchAllDocsQuery();
-            String type = context.getMapperService().documentMapper().type();
-            if (type != null) {
-                BytesRef typeBytes = new BytesRef(type);
-                if (lowerTerm != null) {
-                    int comp = indexedValueForSearch(lowerTerm).compareTo(typeBytes);
-                    if (comp > 0 || (comp == 0 && includeLower == false)) {
-                        result = new MatchNoDocsQuery("[_type] was lexicographically smaller than lower bound of range");
-                    }
-                }
-                if (upperTerm != null) {
-                    int comp = indexedValueForSearch(upperTerm).compareTo(typeBytes);
-                    if (comp < 0 || (comp == 0 && includeUpper == false)) {
-                        result = new MatchNoDocsQuery("[_type] was lexicographically greater than upper bound of range");
-                    }
-                }
+        protected boolean matches(String pattern, QueryShardContext context) {
+            if (pattern.contains("?") == false) {
+                return Regex.simpleMatch(pattern, MapperService.SINGLE_MAPPING_NAME);
             }
-            return result;
+            boolean matches = Pattern.matches(pattern.replace("?", "."), MapperService.SINGLE_MAPPING_NAME);
+            return matches;
         }
     }
 

server/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java

@@ -59,20 +59,52 @@ public void testTermsQuery() throws Exception {
         Query query = ft.termQuery("my_type", context);
         assertEquals(new MatchNoDocsQuery(), query);
 
-        DocumentMapper mapper = Mockito.mock(DocumentMapper.class);
-        Mockito.when(mapper.type()).thenReturn("my_type");
-        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
-        query = ft.termQuery("my_type", context);
+        query = ft.termQuery("_doc", context);
         assertEquals(new MatchAllDocsQuery(), query);
 
         Mockito.when(mapperService.hasNested()).thenReturn(true);
-        query = ft.termQuery("my_type", context);
+        query = ft.termQuery("_doc", context);
         assertEquals(Queries.newNonNestedFilter(), query);
 
-        mapper = Mockito.mock(DocumentMapper.class);
-        Mockito.when(mapper.type()).thenReturn("other_type");
-        Mockito.when(mapperService.documentMapper()).thenReturn(mapper);
         query = ft.termQuery("my_type", context);
         assertEquals(new MatchNoDocsQuery(), query);
     }
+
+    public void testPrefixAndWildCardQuery() throws Exception {
+        QueryShardContext context = Mockito.mock(QueryShardContext.class);
+        Version indexVersionCreated = VersionUtils.randomIndexCompatibleVersion(random());
+        Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build();
+        IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY);
+        Mockito.when(context.getIndexSettings()).thenReturn(mockSettings);
+        Mockito.when(context.indexVersionCreated()).thenReturn(indexVersionCreated);
+
+        MapperService mapperService = Mockito.mock(MapperService.class);
+        Mockito.when(mapperService.documentMapper()).thenReturn(null);
+        Mockito.when(context.getMapperService()).thenReturn(mapperService);
+
+        TypeFieldMapper.TypeFieldType ft = new TypeFieldMapper.TypeFieldType();
+        ft.setName(TypeFieldMapper.NAME);
+        Query query = ft.prefixQuery("_do", null, context);
+        assertEquals(new MatchAllDocsQuery(), query);
+
+        query = ft.prefixQuery("_fo", null, context);
+        assertEquals(new MatchNoDocsQuery(), query);
+
+        ft.setName(TypeFieldMapper.NAME);
+        query = ft.wildcardQuery("_d*", null, context);
+        assertEquals(new MatchAllDocsQuery(), query);
+
+        ft.setName(TypeFieldMapper.NAME);
+        query = ft.wildcardQuery("_d?c", null, context);
+        assertEquals(new MatchAllDocsQuery(), query);
+
+        ft.setName(TypeFieldMapper.NAME);
+        query = ft.wildcardQuery("_fo*", null, context);
+        assertEquals(new MatchNoDocsQuery(), query);
+    }
 }

server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java

@@ -556,13 +556,6 @@ public void testParseRelation() {
         assertEquals(ShapeRelation.INTERSECTS, builder.relation());
     }
 
-    public void testTypeField() throws IOException {
-        RangeQueryBuilder builder = QueryBuilders.rangeQuery("_type")
-            .from("value1");
-        builder.doToQuery(createShardContext());
-        assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
-    }
-
     /**
      * Range queries should generally be cacheable, at least the ones we create randomly.
      * This test makes sure we also test the non-cacheable cases regularly.

server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java

@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -75,7 +76,9 @@ protected void doAssertLuceneQuery(WildcardQueryBuilder queryBuilder, Query quer
 
             assertThat(wildcardQuery.getField(), equalTo(expectedFieldName));
             assertThat(wildcardQuery.getTerm().field(), equalTo(expectedFieldName));
-            assertThat(wildcardQuery.getTerm().text(), equalTo(queryBuilder.value()));
+            // wildcard queries get normalized
+            String text = wildcardQuery.getTerm().text().toLowerCase(Locale.ROOT);
+            assertThat(text, equalTo(text));
         } else {
             Query expected = new MatchNoDocsQuery("unknown field [" + expectedFieldName + "]");
             assertEquals(expected, query);
@@ -138,14 +141,14 @@ public void testTypeField() throws IOException {
         builder.doToQuery(createShardContext());
         assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
     }
-    
+
     public void testRewriteIndexQueryToMatchNone() throws IOException {
         WildcardQueryBuilder query = new WildcardQueryBuilder("_index", "does_not_exist");
         QueryShardContext queryShardContext = createShardContext();
         QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
-    }   
-    
+    }
+
     public void testRewriteIndexQueryNotMatchNone() throws IOException {
         String fullIndexName = getIndex().getName();
         String firstHalfOfIndexName = fullIndexName.substring(0,fullIndexName.length()/2);