Skip to content

Extend Approximation Framework to other numeric types #18530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 4, 2025
+844 −470
Merged

Extend Approximation Framework to other numeric types #18530

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ca907ed
Extend approximation to other numeric types
prudhvigodithi Jun 16, 2025
95f25cc
Fix basic tests
prudhvigodithi Jun 17, 2025
a6e8e70
Fix basic tests
prudhvigodithi Jun 17, 2025
a47899c
Add Parameterized tests for all numeric types
prudhvigodithi Jun 23, 2025
a345773
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jun 23, 2025
7f5af90
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jun 24, 2025
df49d6e
Fix FieldSortIT
prudhvigodithi Jun 24, 2025
95dfda6
Fix FieldSortIT
prudhvigodithi Jun 24, 2025
838a84c
Address comments
prudhvigodithi Jun 28, 2025
c60430f
Update tests and changelog
prudhvigodithi Jul 2, 2025
56870fe
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jul 2, 2025
4023e05
Update tests and changelog
prudhvigodithi Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 6 additions & 11 deletions server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -934,7 +934,7 @@ public void testSortMissingNumbers() throws Exception {
indexRandomForConcurrentSearch("test");

// DOUBLE
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC))
Expand All @@ -944,7 +944,6 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -983,7 +982,7 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// FLOAT
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("d_value").order(SortOrder.ASC))
Expand All @@ -993,7 +992,6 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -1032,7 +1030,7 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// UNSIGNED_LONG
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("u_value").order(SortOrder.ASC))
Expand All @@ -1042,7 +1040,6 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -1138,7 +1135,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
indexRandomForConcurrentSearch("test");

// LONG
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("l_value").order(SortOrder.ASC))
Expand Down Expand Up @@ -1177,7 +1174,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// FLOAT
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("d_value").order(SortOrder.ASC))
Expand All @@ -1187,7 +1184,6 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand All @@ -1214,7 +1210,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// UNSIGNED_LONG
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("u_value").order(SortOrder.ASC))
Expand All @@ -1225,7 +1221,6 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
// The order here could be unstable (depends on document order) since missing == field value
assertThat(searchResponse.getHits().getAt(1).getId(), is(oneOf("3", "2")));
assertThat(searchResponse.getHits().getAt(2).getId(), is(oneOf("2", "3")));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down
191 changes: 148 additions & 43 deletions server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ public class NumberFieldMapper extends ParametrizedFieldMapper {

public static final Setting<Boolean> COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", true, Property.IndexScope);

private static final int APPROX_QUERY_NUMERIC_DIMS = 1;

private static NumberFieldMapper toType(FieldMapper in) {
return (NumberFieldMapper) in;
}
Expand Down Expand Up @@ -356,23 +358,50 @@ public Query rangeQuery(
}
u = HalfFloatPoint.nextDown(u);
}
if (isSearchable && hasDocValues) {
Query query = HalfFloatPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(

Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u)
)
: null;
if (isSearchable) {
Query pointRangeQuery = HalfFloatPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u),
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
NumberType.HALF_FLOAT.encodePoint(l),
NumberType.HALF_FLOAT.encodePoint(u),
APPROX_QUERY_NUMERIC_DIMS,
ApproximatePointRangeQuery.HALF_FLOAT_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u)
HalfFloatPoint.halfFloatToSortableShort(u),
dvQuery
);
}
return HalfFloatPoint.newRangeQuery(field, l, u);
return dvQuery;
}

@Override
Expand Down Expand Up @@ -503,23 +532,52 @@ public Query rangeQuery(
u = FloatPoint.nextDown(u);
}
}
if (isSearchable && hasDocValues) {
Query query = FloatPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(

Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u)
)
: null;

if (isSearchable) {
Query pointRangeQuery = FloatPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u),
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
FloatPoint.pack(new float[] { l }).bytes,
FloatPoint.pack(new float[] { u }).bytes,
APPROX_QUERY_NUMERIC_DIMS,
ApproximatePointRangeQuery.FLOAT_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(

if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u)
NumericUtils.floatToSortableInt(u),
dvQuery
);
}
return FloatPoint.newRangeQuery(field, l, u);
return dvQuery;
}

@Override
Expand Down Expand Up @@ -628,23 +686,49 @@ public Query rangeQuery(
QueryShardContext context
) {
return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
if (isSearchable && hasDocValues) {
Query query = DoublePoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u)
)
: null;
if (isSearchable) {
Query pointRangeQuery = DoublePoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u),
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
DoublePoint.pack(new double[] { l }).bytes,
DoublePoint.pack(new double[] { u }).bytes,
APPROX_QUERY_NUMERIC_DIMS,
ApproximatePointRangeQuery.DOUBLE_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u)
NumericUtils.doubleToSortableLong(u),
dvQuery
);
}
return DoublePoint.newRangeQuery(field, l, u);
return dvQuery;
});
}

Expand Down Expand Up @@ -988,23 +1072,33 @@ public Query rangeQuery(
--u;
}
}
if (isSearchable && hasDocValues) {
Query query = IntPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
Query dvQuery = hasDocValues ? SortedNumericDocValuesField.newSlowRangeQuery(field, l, u) : null;
if (isSearchable) {
Query pointRangeQuery = IntPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
}
} else {
query = pointRangeQuery;
}
return query;
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
IntPoint.pack(new int[] { l }).bytes,
IntPoint.pack(new int[] { u }).bytes,
APPROX_QUERY_NUMERIC_DIMS,
ApproximatePointRangeQuery.INT_FORMAT
)
);
}
if (hasDocValues) {
Query query = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
}
return query;
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery);
}
return IntPoint.newRangeQuery(field, l, u);
return dvQuery;
}

@Override
Expand Down Expand Up @@ -1136,11 +1230,10 @@ public Query rangeQuery(
field,
LongPoint.pack(new long[] { l }).bytes,
LongPoint.pack(new long[] { u }).bytes,
new long[] { l }.length,
APPROX_QUERY_NUMERIC_DIMS,
ApproximatePointRangeQuery.LONG_FORMAT
)
);

}
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery);
Expand Down Expand Up @@ -1257,10 +1350,22 @@ public Query rangeQuery(
QueryShardContext context
) {
return unsignedLongRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
if (isSearchable && hasDocValues) {
if (isSearchable) {
Query query = BigIntegerPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
return new IndexOrDocValuesQuery(query, dvQuery);
if (hasDocValues) {
Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
NumberType.UNSIGNED_LONG.encodePoint(l),
NumberType.UNSIGNED_LONG.encodePoint(u),
APPROX_QUERY_NUMERIC_DIMS,
ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT
)
);
}
if (hasDocValues) {
return SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
Expand Down
Loading
Loading