Solving wildcard sorting issue for doc_values

CHANGELOG.md

@@ -7,6 +7,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 ### Changed
+- Disable pruning for `doc_values` for the wildcard field mapper ([#18568](https://github.com/opensearch-project/OpenSearch/pull/18568))
+- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))
+- Update SecureAuxTransportSettingsProvider to distinguish between aux transport types ([#18616](https://github.com/opensearch-project/OpenSearch/pull/18616))
+- Make node duress values cacheable ([#18649](https://github.com/opensearch-project/OpenSearch/pull/18649))
+- Change default value of remote_data_ratio, which is used in Searchable Snapshots and Writeable Warm from 0 to 5 and min allowed value to 1 ([#18767](https://github.com/opensearch-project/OpenSearch/pull/18767))
+- Making multi rate limiters in repository dynamic [#18069](https://github.com/opensearch-project/OpenSearch/pull/18069)
+- Optimize grouping for segment concurrent search by ensuring that documents within each group are as equal as possible ([#18451](https://github.com/opensearch-project/OpenSearch/pull/18451))
+- Move transport-grpc from a core plugin to a module ([#18897](https://github.com/opensearch-project/OpenSearch/pull/18897))
+- Remove `experimental` designation from transport-grpc settings ([#18915](https://github.com/opensearch-project/OpenSearch/pull/18915))
+- Rename package org.opensearch.plugin,transport.grpc to org.opensearch.transport.grpc ([#18923](https://github.com/opensearch-project/OpenSearch/pull/18923))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))

rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml

@@ -393,3 +393,23 @@ setup:
             terms: { my_field: [ "\\*" ] }
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "9" }
+
+---
+"sort on doc value enabled wildcard":
+  - skip:
+      version: " - 3.1.99"
+      reason: "sorting on doc value enabled wildcard has bug before 3.2.0"
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            wildcard:
+              my_field:
+                value: "*"
+          sort:
+            - my_field.doc_values:
+                order: asc
+          size: 4
+  - match: { hits.total.value: 8 }
+  - match: { hits.hits.0._id: "8" }

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -92,6 +92,7 @@
 import org.opensearch.index.analysis.AnalyzerScope;
 import org.opensearch.index.analysis.NamedAnalyzer;
 import org.opensearch.index.fielddata.IndexFieldData;
+import org.opensearch.index.fielddata.plain.NonPruningSortedSetOrdinalsIndexFieldData.NonPruningSortField;
 import org.opensearch.search.sort.SortedWiderNumericSortField;
 
 import java.io.IOException;
@@ -578,7 +579,7 @@ public static void writeSortField(StreamOutput out, SortField sortField) throws
             sortField = newSortField;
         }
 
-        if (sortField.getClass() != SortField.class) {
+        if (sortField.getClass() != SortField.class && sortField.getClass() != NonPruningSortField.class) {
             throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]");
         }
         if (sortField.getField() == null) {
@@ -588,8 +589,8 @@ public static void writeSortField(StreamOutput out, SortField sortField) throws
             out.writeString(sortField.getField());
         }
         if (sortField.getComparatorSource() != null) {
-            IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) sortField
-                .getComparatorSource();
+            IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) (sortField
+                .getComparatorSource());
             writeSortType(out, comparatorSource.reducedType());
             writeMissingValue(out, comparatorSource.missingValue(sortField.getReverse()));
         } else {

server/src/main/java/org/opensearch/index/fielddata/plain/NonPruningSortedSetOrdinalsIndexFieldData.java

@@ -0,0 +1,228 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.fielddata.plain;
+
+import org.apache.lucene.index.IndexSorter;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Pruning;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.Nullable;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.index.fielddata.IndexFieldData;
+import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.opensearch.index.fielddata.IndexFieldDataCache;
+import org.opensearch.index.fielddata.ScriptDocValues;
+import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.opensearch.index.mapper.WildcardFieldMapper;
+import org.opensearch.search.MultiValueMode;
+import org.opensearch.search.aggregations.support.ValuesSourceType;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.function.Function;
+
+/**
+ * Wrapper for {@link SortedSetOrdinalsIndexFieldData} which disables pruning optimization for
+ * sorting. Used in {@link WildcardFieldMapper}.
+ *
+ * @opensearch.internal
+ */
+public class NonPruningSortedSetOrdinalsIndexFieldData extends SortedSetOrdinalsIndexFieldData {
+
+    /**
+     * Builder for non-pruning sorted set ordinals
+     *
+     * @opensearch.internal
+     */
+    public static class Builder implements IndexFieldData.Builder {
+        private final String name;
+        private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction;
+        private final ValuesSourceType valuesSourceType;
+
+        public Builder(String name, ValuesSourceType valuesSourceType) {
+            this(name, AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION, valuesSourceType);
+        }
+
+        public Builder(String name, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction, ValuesSourceType valuesSourceType) {
+            this.name = name;
+            this.scriptFunction = scriptFunction;
+            this.valuesSourceType = valuesSourceType;
+        }
+
+        @Override
+        public NonPruningSortedSetOrdinalsIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) {
+            return new NonPruningSortedSetOrdinalsIndexFieldData(cache, name, valuesSourceType, breakerService, scriptFunction);
+        }
+    }
+
+    public NonPruningSortedSetOrdinalsIndexFieldData(
+        IndexFieldDataCache cache,
+        String fieldName,
+        ValuesSourceType valuesSourceType,
+        CircuitBreakerService breakerService,
+        Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction
+    ) {
+        super(cache, fieldName, valuesSourceType, breakerService, scriptFunction);
+    }
+
+    @Override
+    public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+        /*
+        Check if we can use a simple {@link SortedSetSortField} compatible with index sorting and
+        returns a custom sort field otherwise.
+        */
+        if (nested != null
+            || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN)
+            || (source.sortMissingLast(missingValue) == false && source.sortMissingFirst(missingValue) == false)) {
+            return new NonPruningSortField(new SortField(getFieldName(), source, reverse));
+        }
+        SortField sortField = new NonPruningSortField(
+            new SortedSetSortField(
+                getFieldName(),
+                reverse,
+                sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN
+            )
+        );
+        sortField.setMissingValue(
+            source.sortMissingLast(missingValue) ^ reverse ? SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST
+        );
+        return sortField;
+    }
+
+    /**
+     * {@link SortField} extension which disables pruning in the comparator.
+     *
+     * @opensearch.internal
+     */
+    public static class NonPruningSortField extends SortField {
+        private final SortField delegate;
+
+        private NonPruningSortField(SortField sortField) {
+            super(sortField.getField(), sortField.getType());
+            this.delegate = sortField;
+        }
+
+        public static Type readType(DataInput in) throws IOException {
+            return SortField.readType(in);
+        }
+
+        @Override
+        public Object getMissingValue() {
+            return delegate.getMissingValue();
+        }
+
+        @Override
+        public void setMissingValue(Object missingValue) {
+            delegate.setMissingValue(missingValue);
+        }
+
+        @Override
+        public String getField() {
+            return delegate.getField();
+        }
+
+        @Override
+        public Type getType() {
+            return delegate.getType();
+        }
+
+        @Override
+        public boolean getReverse() {
+            return delegate.getReverse();
+        }
+
+        @Override
+        public FieldComparatorSource getComparatorSource() {
+            FieldComparatorSource source = delegate.getComparatorSource();
+            return new FieldComparatorSource() {
+                @Override
+                public FieldComparator<?> newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) {
+                    return source.newComparator(fieldname, numHits, Pruning.NONE, reversed); // < ----------- ALWAYS DISABLE PRUNING
+                }
+            };
+        }
+
+        @Override
+        public String toString() {
+            return delegate.toString();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            return delegate.equals(o);
+        }
+
+        @Override
+        public int hashCode() {
+            return delegate.hashCode();
+        }
+
+        @Override
+        public void setBytesComparator(Comparator<BytesRef> b) {
+            delegate.setBytesComparator(b);
+        }
+
+        @Override
+        public Comparator<BytesRef> getBytesComparator() {
+            return delegate.getBytesComparator();
+        }
+
+        @Override
+        public FieldComparator<?> getComparator(int numHits, Pruning pruning) {
+            return delegate.getComparator(numHits, Pruning.NONE); // < ----------- ALWAYS DISABLE PRUNING
+        }
+
+        @Override
+        public SortField rewrite(IndexSearcher searcher) throws IOException {
+            return delegate.rewrite(searcher);
+        }
+
+        @Override
+        public boolean needsScores() {
+            return delegate.needsScores();
+        }
+
+        @Override
+        public IndexSorter getIndexSorter() {
+            return delegate.getIndexSorter();
+        }
+
+        @Deprecated
+        @Override
+        public void setOptimizeSortWithIndexedData(boolean optimizeSortWithIndexedData) {
+            delegate.setOptimizeSortWithIndexedData(optimizeSortWithIndexedData);
+        }
+
+        @Deprecated
+        @Override
+        public boolean getOptimizeSortWithIndexedData() {
+            return delegate.getOptimizeSortWithIndexedData();
+        }
+
+        @Deprecated
+        @Override
+        public void setOptimizeSortWithPoints(boolean optimizeSortWithPoints) {
+            delegate.setOptimizeSortWithPoints(optimizeSortWithPoints);
+        }
+
+        @Deprecated
+        @Override
+        public boolean getOptimizeSortWithPoints() {
+            return delegate.getOptimizeSortWithPoints();
+        }
+    }
+}

server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java

@@ -47,7 +47,7 @@
 import org.opensearch.index.analysis.IndexAnalyzers;
 import org.opensearch.index.analysis.NamedAnalyzer;
 import org.opensearch.index.fielddata.IndexFieldData;
-import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
+import org.opensearch.index.fielddata.plain.NonPruningSortedSetOrdinalsIndexFieldData;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.aggregations.support.CoreValuesSourceType;
@@ -366,7 +366,7 @@ NamedAnalyzer normalizer() {
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();
-            return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES);
+            return new NonPruningSortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES);
         }
 
         @Override

server/src/test/java/org/opensearch/index/fielddata/plain/NonPruningSortedSetOrdinalsIndexFieldDataTests.java

@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.fielddata.plain;
+
+import org.apache.lucene.search.Pruning;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSortField;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.fielddata.IndexFieldData;
+import org.opensearch.index.fielddata.IndexFieldDataService;
+import org.opensearch.index.fielddata.plain.NonPruningSortedSetOrdinalsIndexFieldData.NonPruningSortField;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.Mapper.BuilderContext;
+import org.opensearch.index.mapper.WildcardFieldMapper;
+import org.opensearch.indices.IndicesService;
+import org.opensearch.search.MultiValueMode;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+
+import java.io.IOException;
+
+public class NonPruningSortedSetOrdinalsIndexFieldDataTests extends OpenSearchSingleNodeTestCase {
+    public void testNonPruningSortedSetOrdinalsIndexFieldData() throws IOException {
+        final IndexService indexService = createIndex("test");
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexFieldDataService ifdService = new IndexFieldDataService(
+            indexService.getIndexSettings(),
+            indicesService.getIndicesFieldDataCache(),
+            indicesService.getCircuitBreakerService(),
+            indexService.mapperService(),
+            indexService.getThreadPool()
+        );
+        final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
+        final MappedFieldType stringMapper = new WildcardFieldMapper.Builder("string").docValues(true).build(ctx).fieldType();
+        ifdService.clear();
+        IndexFieldData<?> fd = ifdService.getForField(stringMapper, "test", () -> { throw new UnsupportedOperationException(); });
+        assertTrue(fd instanceof NonPruningSortedSetOrdinalsIndexFieldData);
+        SortField field = ((NonPruningSortedSetOrdinalsIndexFieldData) fd).sortField(null, MultiValueMode.MAX, null, false);
+        assertTrue(field instanceof NonPruningSortField);
+
+        // Test all the methods of NonPruningSortField for code coverage
+
+        field.setMissingValue(SortedSetSortField.STRING_FIRST);
+        field.getMissingValue();
+        field.getField();
+        field.getType();
+        assertFalse(field.getReverse());
+        field.getComparatorSource();
+        field.toString();
+        field.hashCode();
+        field.equals(field);
+        field.getBytesComparator();
+        field.getComparator(0, Pruning.NONE);
+        field.needsScores();
+        field.getIndexSorter();
+        field.setOptimizeSortWithIndexedData(false);
+        field.getOptimizeSortWithIndexedData();
+        field.setOptimizeSortWithPoints(false);
+        field.getOptimizeSortWithPoints();
+    }
+}