Add skip_list parameter to Numeric Field Mappers (default false)

CHANGELOG.md

@@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - APIs for stream transport and new stream-based search api action ([#18722](https://github.com/opensearch-project/OpenSearch/pull/18722))
 - Added the core process for warming merged segments in remote-store enabled domains ([#18683](https://github.com/opensearch-project/OpenSearch/pull/18683))
 - Optimize Composite Aggregations by removing unnecessary object allocations ([#18531](https://github.com/opensearch-project/OpenSearch/pull/18531))
+- Add skip_list parameter to Numeric Field Mappers (default false) ([#18889](https://github.com/opensearch-project/OpenSearch/pull/18889))
 
 ### Changed
 - Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -468,7 +468,14 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
         long scaledValue = Math.round(doubleValue * scalingFactor);
 
-        List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), scaledValue, indexed, hasDocValues, stored);
+        List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
+            fieldType().name(),
+            scaledValue,
+            indexed,
+            hasDocValues,
+            false,
+            stored
+        );
         context.doc().addAll(fields);
 
         if (hasDocValues == false && (indexed || stored)) {

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java

@@ -173,7 +173,8 @@ protected void parseCreateField(ParseContext context) throws IOException {
             tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
         }
 
-        context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, store));
+        context.doc()
+            .addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, false, store));
     }
 
     /**

modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java

@@ -321,7 +321,7 @@ public void testDuel() throws Exception {
             document.add(new TextField(entry.getKey(), value, Field.Store.NO));
         }
         for (Integer intValue : intValues) {
-            List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false);
+            List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false, false);
             for (Field numberField : numberFields) {
                 document.add(numberField);
             }
@@ -449,6 +449,7 @@ public void testDuel2() throws Exception {
                 between(range[0], range[1]),
                 true,
                 true,
+                false,
                 false
             );
             for (Field numberField : numberFields) {

plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java

@@ -99,7 +99,7 @@ public void postParse(ParseContext context) throws IOException {
             return;
         }
         final int value = context.sourceToParse().source().length();
-        context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, true));
+        context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));
     }
 
     @Override

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

@@ -58,6 +58,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.support.XContentMapValues;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
@@ -118,6 +119,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
         private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
         private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
+        private final Parameter<Boolean> skiplist = new Parameter<>(
+            "skip_list",
+            false,
+            () -> false,
+            (n, c, o) -> XContentMapValues.nodeBooleanValue(o),
+            m -> toType(m).skiplist
+        );
 
         private final Parameter<Explicit<Boolean>> ignoreMalformed;
         private final Parameter<Explicit<Boolean>> coerce;
@@ -169,7 +177,7 @@ public Builder docValues(boolean hasDocValues) {
 
         @Override
         protected List<Parameter<?>> getParameters() {
-            return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, meta);
+            return Arrays.asList(indexed, hasDocValues, stored, skiplist, ignoreMalformed, coerce, nullValue, meta);
         }
 
         @Override
@@ -405,13 +413,26 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
                 List<Field> fields = new ArrayList<>();
                 if (indexed) {
                     fields.add(new HalfFloatPoint(name, value.floatValue()));
                 }
                 if (docValued) {
-                    fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
+                    if (skiplist) {
+                        fields.add(
+                            SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue()))
+                        );
+                    } else {
+                        fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
+                    }
                 }
                 if (stored) {
                     fields.add(new StoredField(name, value.floatValue()));
@@ -581,13 +602,24 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
                 List<Field> fields = new ArrayList<>();
                 if (indexed) {
                     fields.add(new FloatPoint(name, value.floatValue()));
                 }
                 if (docValued) {
-                    fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
+                    if (skiplist) {
+                        fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(value.floatValue())));
+                    } else {
+                        fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
+                    }
                 }
                 if (stored) {
                     fields.add(new StoredField(name, value.floatValue()));
@@ -733,13 +765,24 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
                 List<Field> fields = new ArrayList<>();
                 if (indexed) {
                     fields.add(new DoublePoint(name, value.doubleValue()));
                 }
                 if (docValued) {
-                    fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
+                    if (skiplist) {
+                        fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
+                    } else {
+                        fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
+                    }
                 }
                 if (stored) {
                     fields.add(new StoredField(name, value.doubleValue()));
@@ -828,8 +871,15 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
-                return INTEGER.createFields(name, value, indexed, docValued, stored);
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
+                return INTEGER.createFields(name, value, indexed, docValued, skiplist, stored);
             }
 
             @Override
@@ -908,8 +958,15 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
-                return INTEGER.createFields(name, value, indexed, docValued, stored);
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
+                return INTEGER.createFields(name, value, indexed, docValued, skiplist, stored);
             }
 
             @Override
@@ -1102,13 +1159,24 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
                 List<Field> fields = new ArrayList<>();
                 if (indexed) {
                     fields.add(new IntPoint(name, value.intValue()));
                 }
                 if (docValued) {
-                    fields.add(new SortedNumericDocValuesField(name, value.intValue()));
+                    if (skiplist) {
+                        fields.add(SortedNumericDocValuesField.indexedField(name, value.intValue()));
+                    } else {
+                        fields.add(new SortedNumericDocValuesField(name, value.intValue()));
+                    }
                 }
                 if (stored) {
                     fields.add(new StoredField(name, value.intValue()));
@@ -1244,13 +1312,24 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
                 List<Field> fields = new ArrayList<>();
                 if (indexed) {
                     fields.add(new LongPoint(name, value.longValue()));
                 }
                 if (docValued) {
-                    fields.add(new SortedNumericDocValuesField(name, value.longValue()));
+                    if (skiplist) {
+                        fields.add(SortedNumericDocValuesField.indexedField(name, value.longValue()));
+                    } else {
+                        fields.add(new SortedNumericDocValuesField(name, value.longValue()));
+                    }
                 }
                 if (stored) {
                     fields.add(new StoredField(name, value.longValue()));
@@ -1375,7 +1454,14 @@ public Query rangeQuery(
             }
 
             @Override
-            public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
+            public List<Field> createFields(
+                String name,
+                Number value,
+                boolean indexed,
+                boolean docValued,
+                boolean skiplist,
+                boolean stored
+            ) {
                 List<Field> fields = new ArrayList<>();
                 final BigInteger v = Numbers.toUnsignedLongExact(value);
 
@@ -1384,7 +1470,11 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
                 }
 
                 if (docValued) {
-                    fields.add(new SortedNumericDocValuesField(name, v.longValue()));
+                    if (skiplist) {
+                        fields.add(SortedNumericDocValuesField.indexedField(name, v.longValue()));
+                    } else {
+                        fields.add(new SortedNumericDocValuesField(name, v.longValue()));
+                    }
                 }
 
                 if (stored) {
@@ -1453,7 +1543,14 @@ public abstract Query rangeQuery(
 
         public abstract Number parsePoint(byte[] value);
 
-        public abstract List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored);
+        public abstract List<Field> createFields(
+            String name,
+            Number value,
+            boolean indexed,
+            boolean docValued,
+            boolean skiplist,
+            boolean stored
+        );
 
         abstract Number valueForSearch(String value);
 
@@ -1866,6 +1963,7 @@ public Number parse(Object value) {
     private final boolean indexed;
     private final boolean hasDocValues;
     private final boolean stored;
+    private final boolean skiplist;
     private final Explicit<Boolean> ignoreMalformed;
     private final Explicit<Boolean> coerce;
     private final Number nullValue;
@@ -1879,6 +1977,7 @@ private NumberFieldMapper(String simpleName, MappedFieldType mappedFieldType, Mu
         this.indexed = builder.indexed.getValue();
         this.hasDocValues = builder.hasDocValues.getValue();
         this.stored = builder.stored.getValue();
+        this.skiplist = builder.skiplist.getValue();
         this.ignoreMalformed = builder.ignoreMalformed.getValue();
         this.coerce = builder.coerce.getValue();
         this.nullValue = builder.nullValue.getValue();
@@ -1947,7 +2046,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
             numericValue = fieldType().type.parse(value, coerce.value());
         }
 
-        context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, stored));
+        context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, skiplist, stored));
 
         if (hasDocValues == false && (stored || indexed)) {
             createFieldNamesField(context);

server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java

@@ -55,7 +55,7 @@ public void testSingleValued() throws IOException {
         // we need the default codec to check for singletons
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec()));
         Document doc = new Document();
-        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
+        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false, false)) {
             doc.add(f);
         }
         w.addDocument(doc);
@@ -74,10 +74,10 @@ public void testMultiValued() throws IOException {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
         Document doc = new Document();
-        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
+        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false, false)) {
             doc.add(f);
         }
-        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false)) {
+        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false, false)) {
             doc.add(f);
         }
         w.addDocument(doc);