ESQL: Support reading points from doc-values for STATS ST_CENTROID

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -189,6 +189,11 @@ public String indexName() {
                     return "benchmark";
                 }
 
+                @Override
+                public boolean forStats() {
+                    return false;
+                }
+
                 @Override
                 public SearchLookup lookup() {
                     throw new UnsupportedOperationException();

docs/changelog/104218.yaml

@@ -0,0 +1,5 @@
+pr: 104218
+summary: "Support reading spatial points from doc-values for STATS"
+area: "ES|QL"
+type: enhancement
+issues: []

docs/reference/esql/functions/aggregation-functions.asciidoc

@@ -16,6 +16,7 @@ The <<esql-stats-by>> function supports these aggregate functions:
 * <<esql-agg-median-absolute-deviation>>
 * <<esql-agg-min>>
 * <<esql-agg-percentile>>
+* <<esql-agg-st-centroid>>
 * <<esql-agg-sum>>
 // end::agg_list[]
 
@@ -27,4 +28,5 @@ include::median.asciidoc[]
 include::median-absolute-deviation.asciidoc[]
 include::min.asciidoc[]
 include::percentile.asciidoc[]
+include::st_centroid.asciidoc[]
 include::sum.asciidoc[]

docs/reference/esql/functions/st_centroid.asciidoc

@@ -0,0 +1,18 @@
+[discrete]
+[[esql-agg-st-centroid]]
+=== `ST_CENTROID`
+
+Calculate the spatial centroid over a field with spatial point geometry type.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/spatial.csv-spec[tag=st_centroid-airports]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/spatial.csv-spec[tag=st_centroid-airports-result]
+|===
+
+Supported types:
+
+include::types/st_centroid.asciidoc[]

docs/reference/esql/functions/to_cartesianpoint.asciidoc

@@ -17,3 +17,7 @@ include::{esql-specs}/spatial.csv-spec[tag=to_cartesianpoint-str]
 |===
 include::{esql-specs}/spatial.csv-spec[tag=to_cartesianpoint-str-result]
 |===
+
+Supported types:
+
+include::types/to_cartesianpoint.asciidoc[]

docs/reference/esql/functions/to_geopoint.asciidoc

@@ -17,3 +17,7 @@ include::{esql-specs}/spatial.csv-spec[tag=to_geopoint-str]
 |===
 include::{esql-specs}/spatial.csv-spec[tag=to_geopoint-str-result]
 |===
+
+Supported types:
+
+include::types/to_geopoint.asciidoc[]

docs/reference/esql/functions/types/st_centroid.asciidoc

@@ -0,0 +1,6 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+geo_point | geo_point
+cartesian_point | cartesian_point
+|===

docs/reference/esql/functions/types/to_cartesianpoint.asciidoc

@@ -0,0 +1,7 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+cartesian_point | cartesian_point
+keyword | cartesian_point
+text | cartesian_point
+|===

docs/reference/esql/functions/types/to_geopoint.asciidoc

@@ -0,0 +1,7 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+v | result
+geo_point | geo_point
+keyword | geo_point
+text | geo_point
+|===

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

@@ -265,7 +265,7 @@ protected IngestScriptSupport ingestScriptSupport() {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> ((BytesRef) v).utf8ToString();
     }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

@@ -438,7 +438,7 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> (Number) v;
     }
 

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -143,6 +143,10 @@ protected Object parseSourceValue(Object value) {
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             // Currently we can only load from source in ESQL
+            return blockLoaderFromSource(blContext);
+        }
+
+        protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) {
             ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB);
             // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
             return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll());

server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java

@@ -176,5 +176,13 @@ protected AbstractPointFieldType(
         protected Object nullValueAsSource(T nullValue) {
             return nullValue == null ? null : nullValue.toWKT();
         }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            if (blContext.forStats() && hasDocValues()) {
+                return new BlockDocValuesReader.LongsBlockLoader(name());
+            }
+            return blockLoaderFromSource(blContext);
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

@@ -649,6 +649,14 @@ public interface BlockLoaderContext {
          */
         String indexName();
 
+        /**
+         * Whether the data will be used in stats.
+         * This is relevant to some types, where the choice between doc-values or reading from source is dependent on the usage.
+         * For example, spatial types use doc-values for stats, but read from source for search because doc-values are modified
+         * from the original.
+         */
+        boolean forStats();
+
         /**
          * {@link SearchLookup} used for building scripts.
          */

server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java

@@ -264,7 +264,7 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         // Just assert that we expect a boolean. Otherwise no munging.
         return v -> (Boolean) v;
     }

server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -706,7 +706,7 @@ public void execute() {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> ((Number) v).longValue();
     }
 

server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java

@@ -56,7 +56,7 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> {
             // The test converts the float into a string so we do do
             Number n = (Number) v;

server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java

@@ -598,6 +598,11 @@ public void testScriptAndPrecludedParameters() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
+        return syntheticSourceSupport(ignoreMalformed, false);
+    }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed, boolean columnReader) {
         assumeFalse("synthetic _source for geo_point doesn't support ignore_malformed", ignoreMalformed);
         return new SyntheticSourceSupport() {
             private final boolean ignoreZValue = usually();
@@ -607,6 +612,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
             public SyntheticSourceExample example(int maxVals) {
                 if (randomBoolean()) {
                     Tuple<Object, GeoPoint> v = generateValue();
+                    if (columnReader) {
+                        return new SyntheticSourceExample(v.v1(), decode(encode(v.v2())), encode(v.v2()), this::mapping);
+                    }
                     return new SyntheticSourceExample(v.v1(), v.v2(), v.v2().toWKT(), this::mapping);
                 }
                 List<Tuple<Object, GeoPoint>> values = randomList(1, maxVals, this::generateValue);
@@ -616,12 +624,20 @@ public SyntheticSourceExample example(int maxVals) {
                     .sorted((a, b) -> Long.compare(encode(a.v2()), encode(b.v2())))
                     .toList();
                 List<Object> in = sorted.stream().map(Tuple::v1).toList();
-                List<GeoPoint> outList = sorted.stream().map(v -> encode(v.v2())).sorted().map(this::decode).toList();
+                List<GeoPoint> outList = sorted.stream().map(Tuple::v2).toList();
                 Object out = outList.size() == 1 ? outList.get(0) : outList;
 
-                List<String> outBlockList = outList.stream().map(GeoPoint::toWKT).toList();
-                Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
-                return new SyntheticSourceExample(in, out, outBlock, this::mapping);
+                if (columnReader) {
+                    // When reading doc-values, the block is a list of encoded longs
+                    List<Long> outBlockList = outList.stream().map(this::encode).toList();
+                    Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
+                    return new SyntheticSourceExample(in, out, outBlock, this::mapping);
+                } else {
+                    // When reading row-stride, the block is a list of WKT encoded BytesRefs
+                    List<String> outBlockList = outList.stream().map(GeoPoint::toWKT).toList();
+                    Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
+                    return new SyntheticSourceExample(in, out, outBlock, this::mapping);
+                }
             }
 
             private Tuple<Object, GeoPoint> generateValue() {
@@ -705,13 +721,18 @@ protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
 
-    private boolean useDocValues = false;
+    @Override
+    protected Function<Object, Object> loadBlockExpected() {
+        throw new IllegalStateException("Should never reach here, call loadBlockExpected(BlockReaderSupport, boolean) instead");
+    }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
-        if (useDocValues) {
+    protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        if (columnReader) {
+            // When using column reader, we expect the output to be doc-values (which means encoded longs)
             return v -> asJacksonNumberOutput(((Number) v).longValue());
         } else {
+            // When using row-stride reader, we expect the output to be WKT encoded BytesRef
             return v -> asWKT((BytesRef) v);
         }
     }
@@ -732,13 +753,8 @@ protected static Object asWKT(BytesRef value) {
     }
 
     @Override
-    protected boolean supportsColumnAtATimeReader(MapperService mapper, MappedFieldType ft) {
-        // Currently ESQL support for geo_point is limited to source values
-        return false;
-    }
-
-    @Override
-    public void testBlockLoaderFromRowStrideReaderWithSyntheticSource() {
-        assumeTrue("Synthetic source not completed supported for geo_point", false);
+    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        MappedFieldType ft = mapper.fieldType(loaderFieldName);
+        return new BlockReaderSupport(ft.hasDocValues(), true, false, mapper, loaderFieldName);
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java

@@ -56,7 +56,7 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> {
             // The test converts the float into a string so we do do
             Number n = (Number) v;

server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java

@@ -423,7 +423,7 @@ public void execute() {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> InetAddresses.toAddrString(InetAddressPoint.decode(BytesRef.deepCopyOf((BytesRef) v).bytes));
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

@@ -650,7 +650,7 @@ protected boolean supportsIgnoreMalformed() {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return v -> ((BytesRef) v).utf8ToString();
     }
 

server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java

@@ -122,7 +122,7 @@ public void testFetchCoerced() throws IOException {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return n -> {
             Number number = ((Number) n);
             if (Integer.MIN_VALUE <= number.longValue() && number.longValue() <= Integer.MAX_VALUE) {

server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java

@@ -382,7 +382,7 @@ public void testAllowMultipleValuesField() throws IOException {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String loaderFieldName) {
+    protected Function<Object, Object> loadBlockExpected() {
         return n -> ((Number) n); // Just assert it's a number
     }
 

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -1195,8 +1195,8 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
     }
 
     @Override
-    protected Function<Object, Object> loadBlockExpected(MapperService mapper, String fieldName) {
-        if (nullLoaderExpected(mapper, fieldName)) {
+    protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        if (nullLoaderExpected(blockReaderSupport.mapper(), blockReaderSupport.loaderFieldName())) {
             return null;
         }
         return v -> ((BytesRef) v).utf8ToString();
@@ -1352,20 +1352,22 @@ public void testEmpty() throws Exception {
     }
 
     @Override
-    protected boolean supportsColumnAtATimeReader(MapperService mapper, MappedFieldType ft) {
+    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        MappedFieldType ft = mapper.fieldType(loaderFieldName);
         String parentName = mapper.mappingLookup().parentField(ft.name());
         if (parentName == null) {
             TextFieldMapper.TextFieldType text = (TextFieldType) ft;
-            return text.syntheticSourceDelegate() != null
+            boolean supportsColumnAtATimeReader = text.syntheticSourceDelegate() != null
                 && text.syntheticSourceDelegate().hasDocValues()
                 && text.canUseSyntheticSourceDelegateForQuerying();
+            return new BlockReaderSupport(supportsColumnAtATimeReader, mapper, loaderFieldName);
         }
         MappedFieldType parent = mapper.fieldType(parentName);
         if (false == parent.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
             throw new UnsupportedOperationException();
         }
         KeywordFieldMapper.KeywordFieldType kwd = (KeywordFieldMapper.KeywordFieldType) parent;
-        return kwd.hasDocValues();
+        return new BlockReaderSupport(kwd.hasDocValues(), mapper, loaderFieldName);
     }
 
     public void testBlockLoaderFromParentColumnReader() throws IOException {
@@ -1398,6 +1400,7 @@ private void testBlockLoaderFromParent(boolean columnReader, boolean syntheticSo
             b.endObject();
         };
         MapperService mapper = createMapperService(syntheticSource ? syntheticSourceMapping(buildFields) : mapping(buildFields));
-        testBlockLoader(columnReader, example, mapper, "field.sub");
+        BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field.sub");
+        testBlockLoader(columnReader, example, blockReaderSupport);
     }
 }

test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java

@@ -422,6 +422,11 @@ public String indexName() {
                 throw new UnsupportedOperationException();
             }
 
+            @Override
+            public boolean forStats() {
+                return false;
+            }
+
             @Override
             public SearchLookup lookup() {
                 return mockContext().lookup();