CSV IT spec

x-pack/plugin/esql/build.gradle

@@ -81,6 +81,13 @@ dependencies {
   internalClusterTestImplementation project(xpackModule('inference:qa:test-service-plugin'))
   internalClusterTestImplementation(testArtifact(project(xpackModule('inference'))))
   internalClusterTestImplementation project(path: xpackModule('mapper-constant-keyword'))
+  internalClusterTestImplementation project(xpackModule('analytics'))
+  internalClusterTestImplementation project(xpackModule('spatial'))
+  internalClusterTestImplementation project(xpackModule('mapper-unsigned-long'))
+  internalClusterTestImplementation project(xpackModule('wildcard'))
+  internalClusterTestImplementation project(xpackModule('mapper-version'))
+  internalClusterTestImplementation project(xpackModule('mapper-aggregate-metric'))
+  internalClusterTestImplementation project(xpackModule('mapper-exponential-histogram'))
 }
 
 tasks.named('splitPackagesAudit').configure {
@@ -286,6 +293,10 @@ tasks.named("test").configure {
   }
 }
 
+tasks.named('internalClusterTestTestingConventions').configure {
+  baseClass 'org.elasticsearch.test.ESTestCase'
+}
+
 // This is similar to the test task above, but needed for the LookupJoinTypesIT which runs in the internalClusterTest task
 // and generates a types table for the LOOKUP JOIN command. It is possible in future we might have move tests that do this.
 tasks.named("internalClusterTest").configure {

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -17,21 +17,13 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.Types;
-import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.geometry.Point;
-import org.elasticsearch.geometry.utils.GeometryValidator;
-import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.TestFeatureService;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
 import org.elasticsearch.xpack.esql.CsvTestUtils;
@@ -51,9 +43,6 @@
 import org.junit.Rule;
 
 import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.MathContext;
-import java.math.RoundingMode;
 import java.net.URL;
 import java.util.Arrays;
 import java.util.List;
@@ -66,14 +55,9 @@
 import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
-import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
-import static org.elasticsearch.xpack.esql.CsvAssert.assertData;
-import static org.elasticsearch.xpack.esql.CsvAssert.assertMetadata;
+import static org.elasticsearch.xpack.esql.CsvAssert.assertResultsWithTransformer;
 import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
@@ -506,74 +490,18 @@ protected void assertResults(
         List<List<Object>> actualValues,
         Logger logger
     ) {
-        assertMetadata(expected, actualColumns, logger);
-        assertData(expected, actualValues, testCase.ignoreOrder, ignoreValueOrder(), logger, this::valueMapper);
-    }
-
-    private Object valueMapper(CsvTestUtils.Type type, Object value) {
-        if (value == null) {
-            return "null";
-        }
-        if (value instanceof CsvTestUtils.Range) {
-            return value;
-        }
-        if (type == CsvTestUtils.Type.GEO_POINT || type == CsvTestUtils.Type.CARTESIAN_POINT) {
-            // Point tests are failing in clustered integration tests because of tiny precision differences at very small scales
-            if (value instanceof String wkt) {
-                try {
-                    Geometry geometry = WellKnownText.fromWKT(GeometryValidator.NOOP, false, wkt);
-                    if (geometry instanceof Point point) {
-                        return normalizedPoint(type, point.getX(), point.getY());
-                    }
-                } catch (Throwable ignored) {}
-            }
-        }
-        if (type == CsvTestUtils.Type.EXPONENTIAL_HISTOGRAM) {
-            if (value instanceof Map<?, ?> map) {
-                return ExponentialHistogramXContent.parseForTesting(Types.<Map<String, Object>>forciblyCast(map));
-            }
-            if (value instanceof String json) {
-                try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
-                    return ExponentialHistogramXContent.parseForTesting(parser);
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-        if (value instanceof List<?> vs) {
-            return vs.stream().map(v -> valueMapper(type, v)).toList();
-        }
-        if (type == CsvTestUtils.Type.DOUBLE && enableRoundingDoubleValuesOnAsserting()) {
-            if (value instanceof Double d) {
-                if (Double.isNaN(d) || Double.isInfinite(d)) {
-                    return d;
-                }
-                return new BigDecimal(d).round(new MathContext(7, RoundingMode.HALF_DOWN)).doubleValue();
-            } else if (value instanceof String s) {
-                if ("NaN".equals(s)) {
-                    return Double.NaN;
-                }
-                return new BigDecimal(s).round(new MathContext(7, RoundingMode.HALF_DOWN)).doubleValue();
-            }
-        }
-        if (type == CsvTestUtils.Type.TEXT || type == CsvTestUtils.Type.KEYWORD || type == CsvTestUtils.Type.SEMANTIC_TEXT) {
-            if (value instanceof String s) {
-                value = s.replaceAll("\\\\n", "\n");
-            }
-        }
-        if (type == CsvTestUtils.Type.DOUBLE) {
-            if (value instanceof String s && "NaN".equals(s)) {
-                return Double.NaN;
-            }
-            return ((Number) value).doubleValue();
-        }
-        if (type == CsvTestUtils.Type.INTEGER) {
-            return ((Number) value).intValue();
-        }
-        if (type == CsvTestUtils.Type.LONG) {
-            return ((Number) value).longValue();
-        }
-        return value.toString();
+        var actualColumnNames = actualColumns.stream().map(c -> c.get("name")).toList();
+        var actualColumnTypes = actualColumns.stream().map(c -> CsvTestUtils.Type.asType(c.get("type"))).toList();
+        assertResultsWithTransformer(
+            expected,
+            actualColumnNames,
+            actualColumnTypes,
+            actualValues,
+            testCase.ignoreOrder,
+            ignoreValueOrder(),
+            enableRoundingDoubleValuesOnAsserting(),
+            logger
+        );
     }
 
     /**
@@ -584,19 +512,6 @@ protected boolean enableRoundingDoubleValuesOnAsserting() {
         return false;
     }
 
-    private static String normalizedPoint(CsvTestUtils.Type type, double x, double y) {
-        if (type == CsvTestUtils.Type.GEO_POINT) {
-            return normalizedGeoPoint(x, y);
-        }
-        return String.format(Locale.ROOT, "POINT (%f %f)", (float) x, (float) y);
-    }
-
-    private static String normalizedGeoPoint(double x, double y) {
-        x = decodeLongitude(encodeLongitude(x));
-        y = decodeLatitude(encodeLatitude(y));
-        return String.format(Locale.ROOT, "POINT (%f %f)", x, y);
-    }
-
     private Throwable reworkException(Throwable th) {
         StackTraceElement[] stackTrace = th.getStackTrace();
         StackTraceElement[] redone = new StackTraceElement[stackTrace.length + 1];

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java

@@ -13,31 +13,48 @@
 import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder;
 import org.elasticsearch.compute.data.LongRangeBlockBuilder;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.Types;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.utils.Geohash;
+import org.elasticsearch.geometry.utils.GeometryValidator;
+import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.h3.H3;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
 import org.elasticsearch.test.ListMatcher;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.CsvTestUtils.ActualResults;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
 import org.elasticsearch.xpack.versionfield.Version;
 import org.hamcrest.Description;
 import org.hamcrest.Matchers;
 import org.hamcrest.StringDescription;
 
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.math.RoundingMode;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.Type;
@@ -61,25 +78,33 @@
 public final class CsvAssert {
     private CsvAssert() {}
 
-    static void assertResults(ExpectedResults expected, ActualResults actual, boolean ignoreOrder, Logger logger) {
-        assertMetadata(expected, actual, logger);
-        assertData(expected, actual, ignoreOrder, logger);
+    public static void assertResultsWithTransformer(
+        ExpectedResults expected,
+        List<String> actualColumnNames,
+        List<CsvTestUtils.Type> actualColumnTypes,
+        List<List<Object>> actualValues,
+        boolean ignoreOrder,
+        boolean ignoreValueOrder,
+        boolean enableRoundingDoubleValuesOnAsserting,
+        Logger logger
+    ) {
+        assertMetadata(expected, actualColumnNames, actualColumnTypes, List.of(), logger);
+        assertData(
+            expected,
+            actualValues,
+            ignoreOrder,
+            ignoreValueOrder,
+            logger,
+            new AssertedValueTransformer(enableRoundingDoubleValuesOnAsserting)
+        );
     }
 
-    static void assertMetadata(ExpectedResults expected, ActualResults actual, Logger logger) {
+    static void assertResults(ExpectedResults expected, ActualResults actual, boolean ignoreOrder, Logger logger) {
         assertMetadata(expected, actual.columnNames(), actual.columnTypes(), actual.pages(), logger);
+        assertData(expected, actual.values(), ignoreOrder, false, logger, (t, v) -> v);
     }
 
-    public static void assertMetadata(ExpectedResults expected, List<Map<String, String>> actualColumns, Logger logger) {
-        var actualColumnNames = new ArrayList<String>(actualColumns.size());
-        var actualColumnTypes = actualColumns.stream()
-            .peek(c -> actualColumnNames.add(c.get("name")))
-            .map(c -> Type.asType(c.get("type")))
-            .toList();
-        assertMetadata(expected, actualColumnNames, actualColumnTypes, List.of(), logger);
-    }
-
-    private static void assertMetadata(
+    public static void assertMetadata(
         ExpectedResults expected,
         List<String> actualNames,
         List<Type> actualTypes,
@@ -185,21 +210,6 @@ private static void assertMetadata(
         }
     }
 
-    static void assertData(ExpectedResults expected, ActualResults actual, boolean ignoreOrder, Logger logger) {
-        assertData(expected, actual.values(), ignoreOrder, false, logger, (t, v) -> v);
-    }
-
-    public static void assertData(
-        ExpectedResults expected,
-        Iterator<Iterator<Object>> actualValuesIterator,
-        boolean ignoreOrder,
-        boolean ignoreValueOrder,
-        Logger logger,
-        BiFunction<Type, Object, Object> valueTransformer
-    ) {
-        assertData(expected, EsqlTestUtils.getValuesList(actualValuesIterator), ignoreOrder, ignoreValueOrder, logger, valueTransformer);
-    }
-
     private record DataFailure(int row, int column, Object expected, Object actual) {}
 
     public static void assertData(
@@ -316,6 +326,89 @@ private static boolean equals(Object expected, Object actual, boolean ignoreValu
         }
     }
 
+    private record AssertedValueTransformer(boolean enableRoundingDoubleValuesOnAsserting) implements BiFunction<Type, Object, Object> {
+
+        @Override
+        public Object apply(Type type, Object value) {
+            if (value == null) {
+                return "null";
+            }
+            if (value instanceof CsvTestUtils.Range) {
+                return value;
+            }
+            if (type == CsvTestUtils.Type.GEO_POINT || type == CsvTestUtils.Type.CARTESIAN_POINT) {
+                // Point tests are failing in clustered integration tests because of tiny precision differences at very small scales
+                if (value instanceof String wkt) {
+                    try {
+                        Geometry geometry = WellKnownText.fromWKT(GeometryValidator.NOOP, false, wkt);
+                        if (geometry instanceof Point point) {
+                            return normalizedPoint(type, point.getX(), point.getY());
+                        }
+                    } catch (Throwable ignored) {}
+                }
+            }
+            if (type == CsvTestUtils.Type.EXPONENTIAL_HISTOGRAM) {
+                if (value instanceof Map<?, ?> map) {
+                    return ExponentialHistogramXContent.parseForTesting(Types.<Map<String, Object>>forciblyCast(map));
+                }
+                if (value instanceof String json) {
+                    try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
+                        return ExponentialHistogramXContent.parseForTesting(parser);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+            if (value instanceof List<?> vs) {
+                return vs.stream().map(v -> apply(type, v)).toList();
+            }
+            if (type == CsvTestUtils.Type.DOUBLE && enableRoundingDoubleValuesOnAsserting) {
+                if (value instanceof Double d) {
+                    if (Double.isNaN(d) || Double.isInfinite(d)) {
+                        return d;
+                    }
+                    return new BigDecimal(d).round(new MathContext(7, RoundingMode.HALF_DOWN)).doubleValue();
+                } else if (value instanceof String s) {
+                    if ("NaN".equals(s)) {
+                        return Double.NaN;
+                    }
+                    return new BigDecimal(s).round(new MathContext(7, RoundingMode.HALF_DOWN)).doubleValue();
+                }
+            }
+            if (type == CsvTestUtils.Type.TEXT || type == CsvTestUtils.Type.KEYWORD || type == CsvTestUtils.Type.SEMANTIC_TEXT) {
+                if (value instanceof String s) {
+                    value = s.replaceAll("\\\\n", "\n");
+                }
+            }
+            if (type == CsvTestUtils.Type.DOUBLE) {
+                if (value instanceof String s && "NaN".equals(s)) {
+                    return Double.NaN;
+                }
+                return ((Number) value).doubleValue();
+            }
+            if (type == CsvTestUtils.Type.INTEGER) {
+                return ((Number) value).intValue();
+            }
+            if (type == CsvTestUtils.Type.LONG) {
+                return ((Number) value).longValue();
+            }
+            return value.toString();
+        }
+
+        private static String normalizedPoint(CsvTestUtils.Type type, double x, double y) {
+            if (type == CsvTestUtils.Type.GEO_POINT) {
+                return normalizedGeoPoint(x, y);
+            }
+            return String.format(Locale.ROOT, "POINT (%f %f)", (float) x, (float) y);
+        }
+
+        private static String normalizedGeoPoint(double x, double y) {
+            x = decodeLongitude(encodeLongitude(x));
+            y = decodeLatitude(encodeLatitude(y));
+            return String.format(Locale.ROOT, "POINT (%f %f)", x, y);
+        }
+    }
+
     private static void dataFailure(
         String description,
         List<DataFailure> dataFailures,