Skip to content

Commit 05f99db

Browse files
asimmahmood1asimmahmood1
committed
Add skip_list parameter to Numeric Field Mappers
- Add skip_list parameter to numeric field mappers (default false) - Add mapping unit tests for skip_list functionality Signed-off-by: Asim Mahmood <[email protected]>
1 parent 82b57a8 commit 05f99db

File tree

12 files changed

+172
-39
lines changed

12 files changed

+172
-39
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4444
- Make GRPC transport extensible to allow plugins to register and expose their own GRPC services ([#18516](https://github.com/opensearch-project/OpenSearch/pull/18516))
4545
- Added approximation support for range queries with now in date field ([#18511](https://github.com/opensearch-project/OpenSearch/pull/18511))
4646
- Upgrade to protobufs 0.6.0 and clean up deprecated TermQueryProtoUtils code ([#18880](https://github.com/opensearch-project/OpenSearch/pull/18880))
47+
- Add skip_list parameter to Numeric Field Mappers (default false) ([#18889](https://github.com/opensearch-project/OpenSearch/pull/18889))
4748

4849
### Changed
4950
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,14 @@ protected void parseCreateField(ParseContext context) throws IOException {
468468
}
469469
long scaledValue = Math.round(doubleValue * scalingFactor);
470470

471-
List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), scaledValue, indexed, hasDocValues, stored);
471+
List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
472+
fieldType().name(),
473+
scaledValue,
474+
indexed,
475+
hasDocValues,
476+
false,
477+
stored
478+
);
472479
context.doc().addAll(fields);
473480

474481
if (hasDocValues == false && (indexed || stored)) {

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ protected void parseCreateField(ParseContext context) throws IOException {
173173
tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
174174
}
175175

176-
context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, store));
176+
context.doc()
177+
.addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, false, store));
177178
}
178179

179180
/**

‎modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ public void testDuel() throws Exception {
321321
document.add(new TextField(entry.getKey(), value, Field.Store.NO));
322322
}
323323
for (Integer intValue : intValues) {
324-
List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false);
324+
List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false, false);
325325
for (Field numberField : numberFields) {
326326
document.add(numberField);
327327
}
@@ -449,6 +449,7 @@ public void testDuel2() throws Exception {
449449
between(range[0], range[1]),
450450
true,
451451
true,
452+
false,
452453
false
453454
);
454455
for (Field numberField : numberFields) {

‎plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public void postParse(ParseContext context) throws IOException {
9999
return;
100100
}
101101
final int value = context.sourceToParse().source().length();
102-
context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, true));
102+
context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));
103103
}
104104

105105
@Override

‎server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 122 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import org.opensearch.common.settings.Setting;
5959
import org.opensearch.common.settings.Setting.Property;
6060
import org.opensearch.common.settings.Settings;
61+
import org.opensearch.common.xcontent.support.XContentMapValues;
6162
import org.opensearch.core.common.bytes.BytesArray;
6263
import org.opensearch.core.xcontent.XContentBuilder;
6364
import org.opensearch.core.xcontent.XContentParser;
@@ -118,6 +119,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
118119
private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
119120
private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
120121
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
122+
private final Parameter<Boolean> skiplist = new Parameter<>(
123+
"skip_list",
124+
false,
125+
() -> false,
126+
(n, c, o) -> XContentMapValues.nodeBooleanValue(o),
127+
m -> toType(m).skiplist
128+
);
121129

122130
private final Parameter<Explicit<Boolean>> ignoreMalformed;
123131
private final Parameter<Explicit<Boolean>> coerce;
@@ -169,7 +177,7 @@ public Builder docValues(boolean hasDocValues) {
169177

170178
@Override
171179
protected List<Parameter<?>> getParameters() {
172-
return Arrays.asList(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, meta);
180+
return Arrays.asList(indexed, hasDocValues, stored, skiplist, ignoreMalformed, coerce, nullValue, meta);
173181
}
174182

175183
@Override
@@ -405,13 +413,26 @@ public Query rangeQuery(
405413
}
406414

407415
@Override
408-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
416+
public List<Field> createFields(
417+
String name,
418+
Number value,
419+
boolean indexed,
420+
boolean docValued,
421+
boolean skiplist,
422+
boolean stored
423+
) {
409424
List<Field> fields = new ArrayList<>();
410425
if (indexed) {
411426
fields.add(new HalfFloatPoint(name, value.floatValue()));
412427
}
413428
if (docValued) {
414-
fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
429+
if (skiplist) {
430+
fields.add(
431+
SortedNumericDocValuesField.indexedField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue()))
432+
);
433+
} else {
434+
fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
435+
}
415436
}
416437
if (stored) {
417438
fields.add(new StoredField(name, value.floatValue()));
@@ -581,13 +602,24 @@ public Query rangeQuery(
581602
}
582603

583604
@Override
584-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
605+
public List<Field> createFields(
606+
String name,
607+
Number value,
608+
boolean indexed,
609+
boolean docValued,
610+
boolean skiplist,
611+
boolean stored
612+
) {
585613
List<Field> fields = new ArrayList<>();
586614
if (indexed) {
587615
fields.add(new FloatPoint(name, value.floatValue()));
588616
}
589617
if (docValued) {
590-
fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
618+
if (skiplist) {
619+
fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.floatToSortableInt(value.floatValue())));
620+
} else {
621+
fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue())));
622+
}
591623
}
592624
if (stored) {
593625
fields.add(new StoredField(name, value.floatValue()));
@@ -733,13 +765,24 @@ public Query rangeQuery(
733765
}
734766

735767
@Override
736-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
768+
public List<Field> createFields(
769+
String name,
770+
Number value,
771+
boolean indexed,
772+
boolean docValued,
773+
boolean skiplist,
774+
boolean stored
775+
) {
737776
List<Field> fields = new ArrayList<>();
738777
if (indexed) {
739778
fields.add(new DoublePoint(name, value.doubleValue()));
740779
}
741780
if (docValued) {
742-
fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
781+
if (skiplist) {
782+
fields.add(SortedNumericDocValuesField.indexedField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
783+
} else {
784+
fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue())));
785+
}
743786
}
744787
if (stored) {
745788
fields.add(new StoredField(name, value.doubleValue()));
@@ -828,8 +871,15 @@ public Query rangeQuery(
828871
}
829872

830873
@Override
831-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
832-
return INTEGER.createFields(name, value, indexed, docValued, stored);
874+
public List<Field> createFields(
875+
String name,
876+
Number value,
877+
boolean indexed,
878+
boolean docValued,
879+
boolean skiplist,
880+
boolean stored
881+
) {
882+
return INTEGER.createFields(name, value, indexed, docValued, skiplist, stored);
833883
}
834884

835885
@Override
@@ -908,8 +958,15 @@ public Query rangeQuery(
908958
}
909959

910960
@Override
911-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
912-
return INTEGER.createFields(name, value, indexed, docValued, stored);
961+
public List<Field> createFields(
962+
String name,
963+
Number value,
964+
boolean indexed,
965+
boolean docValued,
966+
boolean skiplist,
967+
boolean stored
968+
) {
969+
return INTEGER.createFields(name, value, indexed, docValued, skiplist, stored);
913970
}
914971

915972
@Override
@@ -1102,13 +1159,24 @@ public Query rangeQuery(
11021159
}
11031160

11041161
@Override
1105-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
1162+
public List<Field> createFields(
1163+
String name,
1164+
Number value,
1165+
boolean indexed,
1166+
boolean docValued,
1167+
boolean skiplist,
1168+
boolean stored
1169+
) {
11061170
List<Field> fields = new ArrayList<>();
11071171
if (indexed) {
11081172
fields.add(new IntPoint(name, value.intValue()));
11091173
}
11101174
if (docValued) {
1111-
fields.add(new SortedNumericDocValuesField(name, value.intValue()));
1175+
if (skiplist) {
1176+
fields.add(SortedNumericDocValuesField.indexedField(name, value.intValue()));
1177+
} else {
1178+
fields.add(new SortedNumericDocValuesField(name, value.intValue()));
1179+
}
11121180
}
11131181
if (stored) {
11141182
fields.add(new StoredField(name, value.intValue()));
@@ -1244,13 +1312,24 @@ public Query rangeQuery(
12441312
}
12451313

12461314
@Override
1247-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
1315+
public List<Field> createFields(
1316+
String name,
1317+
Number value,
1318+
boolean indexed,
1319+
boolean docValued,
1320+
boolean skiplist,
1321+
boolean stored
1322+
) {
12481323
List<Field> fields = new ArrayList<>();
12491324
if (indexed) {
12501325
fields.add(new LongPoint(name, value.longValue()));
12511326
}
12521327
if (docValued) {
1253-
fields.add(new SortedNumericDocValuesField(name, value.longValue()));
1328+
if (skiplist) {
1329+
fields.add(SortedNumericDocValuesField.indexedField(name, value.longValue()));
1330+
} else {
1331+
fields.add(new SortedNumericDocValuesField(name, value.longValue()));
1332+
}
12541333
}
12551334
if (stored) {
12561335
fields.add(new StoredField(name, value.longValue()));
@@ -1375,7 +1454,14 @@ public Query rangeQuery(
13751454
}
13761455

13771456
@Override
1378-
public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) {
1457+
public List<Field> createFields(
1458+
String name,
1459+
Number value,
1460+
boolean indexed,
1461+
boolean docValued,
1462+
boolean skiplist,
1463+
boolean stored
1464+
) {
13791465
List<Field> fields = new ArrayList<>();
13801466
final BigInteger v = Numbers.toUnsignedLongExact(value);
13811467

@@ -1384,7 +1470,11 @@ public List<Field> createFields(String name, Number value, boolean indexed, bool
13841470
}
13851471

13861472
if (docValued) {
1387-
fields.add(new SortedNumericDocValuesField(name, v.longValue()));
1473+
if (skiplist) {
1474+
fields.add(SortedNumericDocValuesField.indexedField(name, v.longValue()));
1475+
} else {
1476+
fields.add(new SortedNumericDocValuesField(name, v.longValue()));
1477+
}
13881478
}
13891479

13901480
if (stored) {
@@ -1453,7 +1543,14 @@ public abstract Query rangeQuery(
14531543

14541544
public abstract Number parsePoint(byte[] value);
14551545

1456-
public abstract List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored);
1546+
public abstract List<Field> createFields(
1547+
String name,
1548+
Number value,
1549+
boolean indexed,
1550+
boolean docValued,
1551+
boolean skiplist,
1552+
boolean stored
1553+
);
14571554

14581555
abstract Number valueForSearch(String value);
14591556

@@ -1866,6 +1963,7 @@ public Number parse(Object value) {
18661963
private final boolean indexed;
18671964
private final boolean hasDocValues;
18681965
private final boolean stored;
1966+
private final boolean skiplist;
18691967
private final Explicit<Boolean> ignoreMalformed;
18701968
private final Explicit<Boolean> coerce;
18711969
private final Number nullValue;
@@ -1879,6 +1977,7 @@ private NumberFieldMapper(String simpleName, MappedFieldType mappedFieldType, Mu
18791977
this.indexed = builder.indexed.getValue();
18801978
this.hasDocValues = builder.hasDocValues.getValue();
18811979
this.stored = builder.stored.getValue();
1980+
this.skiplist = builder.skiplist.getValue();
18821981
this.ignoreMalformed = builder.ignoreMalformed.getValue();
18831982
this.coerce = builder.coerce.getValue();
18841983
this.nullValue = builder.nullValue.getValue();
@@ -1890,6 +1989,10 @@ boolean coerce() {
18901989
return coerce.value();
18911990
}
18921991

1992+
boolean skiplist() {
1993+
return skiplist;
1994+
}
1995+
18931996
@Override
18941997
protected Explicit<Boolean> ignoreMalformed() {
18951998
return ignoreMalformed;
@@ -1947,7 +2050,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
19472050
numericValue = fieldType().type.parse(value, coerce.value());
19482051
}
19492052

1950-
context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, stored));
2053+
context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, false, stored));
19512054

19522055
if (hasDocValues == false && (stored || indexed)) {
19532056
createFieldNamesField(context);

‎server/src/test/java/org/opensearch/index/fielddata/plain/HalfFloatFielddataTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public void testSingleValued() throws IOException {
5555
// we need the default codec to check for singletons
5656
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec()));
5757
Document doc = new Document();
58-
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
58+
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false, false)) {
5959
doc.add(f);
6060
}
6161
w.addDocument(doc);
@@ -74,10 +74,10 @@ public void testMultiValued() throws IOException {
7474
Directory dir = newDirectory();
7575
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
7676
Document doc = new Document();
77-
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
77+
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false, false)) {
7878
doc.add(f);
7979
}
80-
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false)) {
80+
for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false, false)) {
8181
doc.add(f);
8282
}
8383
w.addDocument(doc);

0 commit comments

Comments
 (0)