Skip to content

Extend Approximation Framework to other numeric types #18530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 4, 2025
+844 −470
Merged

Extend Approximation Framework to other numeric types #18530

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ca907ed
Extend approximation to other numeric types
prudhvigodithi Jun 16, 2025
95f25cc
Fix basic tests
prudhvigodithi Jun 17, 2025
a6e8e70
Fix basic tests
prudhvigodithi Jun 17, 2025
a47899c
Add Parameterized tests for all numeric types
prudhvigodithi Jun 23, 2025
a345773
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jun 23, 2025
7f5af90
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jun 24, 2025
df49d6e
Fix FieldSortIT
prudhvigodithi Jun 24, 2025
95dfda6
Fix FieldSortIT
prudhvigodithi Jun 24, 2025
838a84c
Address comments
prudhvigodithi Jun 28, 2025
c60430f
Update tests and changelog
prudhvigodithi Jul 2, 2025
56870fe
Merge remote-tracking branch 'upstream/main' into lucene-10.2.0
prudhvigodithi Jul 2, 2025
4023e05
Update tests and changelog
prudhvigodithi Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 129 additions & 41 deletions server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,23 +356,50 @@ public Query rangeQuery(
}
u = HalfFloatPoint.nextDown(u);
}
if (isSearchable && hasDocValues) {
Query query = HalfFloatPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(

Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u)
)
: null;
if (isSearchable) {
Query pointRangeQuery = HalfFloatPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u),
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
NumberType.HALF_FLOAT.encodePoint(l),
NumberType.HALF_FLOAT.encodePoint(u),
1,
ApproximatePointRangeQuery.HALF_FLOAT_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(
field,
HalfFloatPoint.halfFloatToSortableShort(l),
HalfFloatPoint.halfFloatToSortableShort(u)
HalfFloatPoint.halfFloatToSortableShort(u),
dvQuery
);
}
return HalfFloatPoint.newRangeQuery(field, l, u);
return dvQuery;
}

@Override
Expand Down Expand Up @@ -503,14 +530,28 @@ public Query rangeQuery(
u = FloatPoint.nextDown(u);
}
}
if (isSearchable && hasDocValues) {
if (isSearchable) {
Query query = FloatPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u)

if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.floatToSortableInt(l),
NumericUtils.floatToSortableInt(u)
);
query = new IndexOrDocValuesQuery(query, dvQuery);
}

return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
FloatPoint.pack(new float[] { l }).bytes,
FloatPoint.pack(new float[] { u }).bytes,
1,
ApproximatePointRangeQuery.FLOAT_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
Expand Down Expand Up @@ -628,23 +669,49 @@ public Query rangeQuery(
QueryShardContext context
) {
return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
if (isSearchable && hasDocValues) {
Query query = DoublePoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(
Query dvQuery = hasDocValues
? SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u)
)
: null;
if (isSearchable) {
Query pointRangeQuery = DoublePoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u),
query
);
}
} else {
query = pointRangeQuery;
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
DoublePoint.pack(new double[] { l }).bytes,
DoublePoint.pack(new double[] { u }).bytes,
1,
ApproximatePointRangeQuery.DOUBLE_FORMAT
)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowRangeQuery(
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(
field,
NumericUtils.doubleToSortableLong(l),
NumericUtils.doubleToSortableLong(u)
NumericUtils.doubleToSortableLong(u),
dvQuery
);
}
return DoublePoint.newRangeQuery(field, l, u);
return dvQuery;
});
}

Expand Down Expand Up @@ -988,23 +1055,33 @@ public Query rangeQuery(
--u;
}
}
if (isSearchable && hasDocValues) {
Query query = IntPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
Query dvQuery = hasDocValues ? SortedNumericDocValuesField.newSlowRangeQuery(field, l, u) : null;
if (isSearchable) {
Query pointRangeQuery = IntPoint.newRangeQuery(field, l, u);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
}
} else {
query = pointRangeQuery;
}
return query;
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
IntPoint.pack(new int[] { l }).bytes,
IntPoint.pack(new int[] { u }).bytes,
1,
ApproximatePointRangeQuery.INT_FORMAT
)
);
}
if (hasDocValues) {
Query query = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u);
if (context.indexSortedOnField(field)) {
query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query);
}
return query;
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery);
}
return IntPoint.newRangeQuery(field, l, u);
return dvQuery;
}

@Override
Expand Down Expand Up @@ -1136,11 +1213,10 @@ public Query rangeQuery(
field,
LongPoint.pack(new long[] { l }).bytes,
LongPoint.pack(new long[] { u }).bytes,
new long[] { l }.length,
1,
ApproximatePointRangeQuery.LONG_FORMAT
)
);

}
if (context.indexSortedOnField(field)) {
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, dvQuery);
Expand Down Expand Up @@ -1257,10 +1333,22 @@ public Query rangeQuery(
QueryShardContext context
) {
return unsignedLongRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> {
if (isSearchable && hasDocValues) {
if (isSearchable) {
Query query = BigIntegerPoint.newRangeQuery(field, l, u);
Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
return new IndexOrDocValuesQuery(query, dvQuery);
if (hasDocValues) {
Query dvQuery = SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
NumberType.UNSIGNED_LONG.encodePoint(l),
NumberType.UNSIGNED_LONG.encodePoint(u),
1,
ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT
)
);
}
if (hasDocValues) {
return SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery(field, l, u);
Expand Down
10 changes: 10 additions & 0 deletions server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@

package org.opensearch.search.approximate;

import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.sandbox.document.BigIntegerPoint;
import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
Expand Down Expand Up @@ -40,6 +45,11 @@
*/
public class ApproximatePointRangeQuery extends ApproximateQuery {
public static final Function<byte[], String> LONG_FORMAT = bytes -> Long.toString(LongPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> INT_FORMAT = bytes -> Integer.toString(IntPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> HALF_FLOAT_FORMAT = bytes -> Float.toString(HalfFloatPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> FLOAT_FORMAT = bytes -> Float.toString(FloatPoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> DOUBLE_FORMAT = bytes -> Double.toString(DoublePoint.decodeDimension(bytes, 0));
public static final Function<byte[], String> UNSIGNED_LONG_FORMAT = bytes -> BigIntegerPoint.decodeDimension(bytes, 0).toString();

private int size;

Expand Down
24 changes: 22 additions & 2 deletions server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,10 +411,20 @@ public void testLongRangeQuery() {

public void testUnsignedLongRangeQuery() {
MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.UNSIGNED_LONG);
Query expected = new IndexOrDocValuesQuery(
Query indexOrDvQuery = new IndexOrDocValuesQuery(
BigIntegerPoint.newRangeQuery("field", BigInteger.valueOf(1), BigInteger.valueOf(3)),
SortedUnsignedLongDocValuesRangeQuery.newSlowRangeQuery("field", BigInteger.valueOf(1), BigInteger.valueOf(3))
);
Query expected = new ApproximateScoreQuery(
indexOrDvQuery,
new ApproximatePointRangeQuery(
"field",
NumberType.UNSIGNED_LONG.encodePoint(BigInteger.valueOf(1)),
NumberType.UNSIGNED_LONG.encodePoint(BigInteger.valueOf(3)),
1,
ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT
)
);
assertEquals(expected, ft.rangeQuery("1", "3", true, true, null, null, null, MOCK_QSC));

MappedFieldType unsearchable = unsearchable();
Expand Down Expand Up @@ -443,14 +453,24 @@ public void testUnsignedLongTermsQuery() {

public void testDoubleRangeQuery() {
MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE);
Query expected = new IndexOrDocValuesQuery(
Query indexOrDvQuery = new IndexOrDocValuesQuery(
DoublePoint.newRangeQuery("field", 1d, 3d),
SortedNumericDocValuesField.newSlowRangeQuery(
"field",
NumericUtils.doubleToSortableLong(1),
NumericUtils.doubleToSortableLong(3)
)
);
Query expected = new ApproximateScoreQuery(
indexOrDvQuery,
new ApproximatePointRangeQuery(
"field",
DoublePoint.pack(new double[] { 1d }).bytes,
DoublePoint.pack(new double[] { 3d }).bytes,
1,
ApproximatePointRangeQuery.DOUBLE_FORMAT
)
);
assertEquals(expected, ft.rangeQuery("1", "3", true, true, null, null, null, MOCK_QSC));

MappedFieldType unsearchable = unsearchable();
Expand Down
11 changes: 9 additions & 2 deletions server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,12 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query,
query
);
} else if (expectedFieldName.equals(INT_FIELD_NAME)) {
assertThat(query, instanceOf(IndexOrDocValuesQuery.class));
query = ((IndexOrDocValuesQuery) query).getIndexQuery();
assertThat(query, instanceOf(ApproximateScoreQuery.class));
Query approximationQuery = ((ApproximateScoreQuery) query).getApproximationQuery();
assertThat(approximationQuery, instanceOf(ApproximateQuery.class));
Query originalQuery = ((ApproximateScoreQuery) query).getOriginalQuery();
assertThat(originalQuery, instanceOf(IndexOrDocValuesQuery.class));
query = ((IndexOrDocValuesQuery) originalQuery).getIndexQuery();
assertThat(query, instanceOf(PointRangeQuery.class));
Integer min = (Integer) queryBuilder.from();
Integer max = (Integer) queryBuilder.to();
Expand Down Expand Up @@ -299,6 +303,9 @@ public void testIllegalArguments() {

public void testToQueryNumericField() throws IOException {
Query parsedQuery = rangeQuery(INT_FIELD_NAME).from(23).to(54).includeLower(true).includeUpper(false).toQuery(createShardContext());
if (parsedQuery instanceof ApproximateScoreQuery) {
parsedQuery = ((ApproximateScoreQuery) parsedQuery).getOriginalQuery();
}
// since age is automatically registered in data, we encode it as numeric
assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class));
parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery();
Expand Down
Loading