Skip to content

Extend Approximation Framework to other numeric types #18530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 4, 2025
+844 −470
Merged

Extend Approximation Framework to other numeric types #18530

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ca907ed
Extend approximation to other numeric types
prudhvigodithi Jun 16, 2025
95f25cc
Fix basic tests
prudhvigodithi Jun 17, 2025
a6e8e70
Fix basic tests
prudhvigodithi Jun 17, 2025
a47899c
Add Parameterized tests for all numeric types
prudhvigodithi Jun 23, 2025
a345773
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jun 23, 2025
7f5af90
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jun 24, 2025
df49d6e
Fix FieldSortIT
prudhvigodithi Jun 24, 2025
95dfda6
Fix FieldSortIT
prudhvigodithi Jun 24, 2025
838a84c
Address comments
prudhvigodithi Jun 28, 2025
c60430f
Update tests and changelog
prudhvigodithi Jul 2, 2025
56870fe
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jul 2, 2025
4023e05
Update tests and changelog
prudhvigodithi Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 6 additions & 11 deletions server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -934,7 +934,7 @@ public void testSortMissingNumbers() throws Exception {
indexRandomForConcurrentSearch("test");

// DOUBLE
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC))
Expand All @@ -944,7 +944,6 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -983,7 +982,7 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// FLOAT
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("d_value").order(SortOrder.ASC))
Expand All @@ -993,7 +992,6 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -1032,7 +1030,7 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// UNSIGNED_LONG
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("u_value").order(SortOrder.ASC))
Expand All @@ -1042,7 +1040,6 @@ public void testSortMissingNumbers() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -1138,7 +1135,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
indexRandomForConcurrentSearch("test");

// LONG
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("l_value").order(SortOrder.ASC))
Expand Down Expand Up @@ -1177,7 +1174,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// FLOAT
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("d_value").order(SortOrder.ASC))
Expand All @@ -1187,7 +1184,6 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("2"));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand All @@ -1214,7 +1210,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3"));

// UNSIGNED_LONG
logger.info("--> sort with no missing (same as missing _last)");
logger.info("--> sort with no missing");
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("u_value").order(SortOrder.ASC))
Expand All @@ -1225,7 +1221,6 @@ public void testSortMissingNumbersMinMax() throws Exception {
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
// The order here could be unstable (depends on document order) since missing == field value
assertThat(searchResponse.getHits().getAt(1).getId(), is(oneOf("3", "2")));
assertThat(searchResponse.getHits().getAt(2).getId(), is(oneOf("2", "3")));

logger.info("--> sort with missing _last");
searchResponse = client().prepareSearch()
Expand Down
170 changes: 129 additions & 41 deletions server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,23 +356,50 @@
}
u = HalfFloatPoint.nextDown(u);
}
if (isSearchable && hasDocValues) {
Query query = HalfFloatPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(

Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u)
)
: null;
if (isSearchable) {
Query pointRangeQuery = HalfFloatPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(

Check warning on line 373 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L373 

Added line #L373 was not covered by tests
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u),

Check warning on line 376 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L375-L376 

Added lines #L375 - L376 were not covered by tests
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
NumberType.HALF_FLOAT.encodePoint(l),
NumberType.HALF_FLOAT.encodePoint(u),
1,
ApproximatePointRangeQuery.HALF_FLOAT_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(

Check warning on line 395 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L395 

Added line #L395 was not covered by tests
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u)
HalfFloatPoint.halfFloatToSortableShort(u),

Check warning on line 398 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L398 

Added line #L398 was not covered by tests
dvQuery
);
}
return HalfFloatPoint.newRangeQuery(field, l, u);
return dvQuery;

Check warning on line 402 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L402 

Added line #L402 was not covered by tests
}

@Override
Expand Down Expand Up @@ -503,14 +530,28 @@
u = FloatPoint.nextDown(u);
}
}
if (isSearchable && hasDocValues) {
if (isSearchable) {
Query query = FloatPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u)

if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u)
);
query = new IndexOrDocValuesQuery(query, dvQuery);
}

return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
FloatPoint.pack(new float[] { l }).bytes,
FloatPoint.pack(new float[] { u }).bytes,
1,
ApproximatePointRangeQuery.FLOAT_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
Expand Down Expand Up @@ -628,23 +669,49 @@
QueryShardContext context
) {
return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
if (isSearchable && hasDocValues) {
Query query = DoublePoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u)
)
: null;
if (isSearchable) {
Query pointRangeQuery = DoublePoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(

Check warning on line 685 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L685 

Added line #L685 was not covered by tests
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u),

Check warning on line 688 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L687-L688 

Added lines #L687 - L688 were not covered by tests
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
DoublePoint.pack(new double[] { l }).bytes,
DoublePoint.pack(new double[] { u }).bytes,
1,
ApproximatePointRangeQuery.DOUBLE_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(

Check warning on line 707 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L707 

Added line #L707 was not covered by tests
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u)
NumericUtils.doubleToSortableLong(u),

Check warning on line 710 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L710 

Added line #L710 was not covered by tests
dvQuery
);
}
return DoublePoint.newRangeQuery(field, l, u);
return dvQuery;

Check warning on line 714 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L714 

Added line #L714 was not covered by tests
});
}

Expand Down Expand Up @@ -988,23 +1055,33 @@
--u;
}
}
if (isSearchable && hasDocValues) {
Query query = IntPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
Query dvQuery = hasDocValues ? SortedNumericDocValuesField.newSlowRangeQuery(field, l, u) : null;
if (isSearchable) {
Query pointRangeQuery = IntPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
}
} else {
query = pointRangeQuery;

Check warning on line 1068 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1068 

Added line #L1068 was not covered by tests
}
return query;
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
IntPoint.pack(new int[] { l }).bytes,
IntPoint.pack(new int[] { u }).bytes,
1,
ApproximatePointRangeQuery.INT_FORMAT
)
);
}
if (hasDocValues) {
Query query = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
}
return query;
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery);

Check warning on line 1082 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1082 

Added line #L1082 was not covered by tests
}
return IntPoint.newRangeQuery(field, l, u);
return dvQuery;

Check warning on line 1084 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1084 

Added line #L1084 was not covered by tests
}

@Override
Expand Down Expand Up @@ -1136,11 +1213,10 @@
field,
LongPoint.pack(new long[] { l }).bytes,
LongPoint.pack(new long[] { u }).bytes,
new long[] { l }.length,
1,
ApproximatePointRangeQuery.LONG_FORMAT
)
);

}
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery);
Expand Down Expand Up @@ -1257,10 +1333,22 @@
QueryShardContext context
) {
return unsignedLongRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
if (isSearchable && hasDocValues) {
if (isSearchable) {
Query query = BigIntegerPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
return new IndexOrDocValuesQuery(query, dvQuery);
if (hasDocValues) {
Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
NumberType.UNSIGNED_LONG.encodePoint(l),
NumberType.UNSIGNED_LONG.encodePoint(u),
1,
ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT
)
);
}
if (hasDocValues) {
return SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
Expand Down
10 changes: 10 additions & 0 deletions server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@

package org.opensearch.search.approximate;

import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.sandbox.document.BigIntegerPoint;
import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
Expand Down Expand Up @@ -40,6 +45,11 @@
*/
public class ApproximatePointRangeQuery extends ApproximateQuery {
public static final Function<byte[], String> LONG_FORMAT = bytes -> Long.toString(LongPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> INT_FORMAT = bytes -> Integer.toString(IntPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> HALF_FLOAT_FORMAT = bytes -> Float.toString(HalfFloatPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> FLOAT_FORMAT = bytes -> Float.toString(FloatPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> DOUBLE_FORMAT = bytes -> Double.toString(DoublePoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> UNSIGNED_LONG_FORMAT = bytes -> BigIntegerPoint.decodeDimension(bytes, 0).toString();

private int size;

Expand Down
Loading
Loading