diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 1caef55748eff..3c466183b16db 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -1035,14 +1035,14 @@ public static Literal randomLiteral(DataType type) { } case TSID_DATA_TYPE -> randomTsId().toBytesRef(); case DENSE_VECTOR -> Arrays.asList(randomArray(10, 10, i -> new Float[10], ESTestCase::randomFloat)); - case EXPONENTIAL_HISTOGRAM -> new WriteableExponentialHistogram(EsqlTestUtils.randomExponentialHistogram()); + case EXPONENTIAL_HISTOGRAM -> EsqlTestUtils.randomExponentialHistogram(); case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" ); }, type); } - private static ExponentialHistogram randomExponentialHistogram() { + public static ExponentialHistogram randomExponentialHistogram() { // TODO(b/133393): allow (index,scale) based zero thresholds as soon as we support them in the block // ideally Replace this with the shared random generation in ExponentialHistogramTestUtils boolean hasNegativeValues = randomBoolean(); @@ -1062,7 +1062,8 @@ private static ExponentialHistogram randomExponentialHistogram() { ExponentialHistogramCircuitBreaker.noop(), rawValues ); - return histo; + // Make the result histogram writeable to allow usage in Literals for testing + return new WriteableExponentialHistogram(histo); } static Version randomVersion() { diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileEvaluator.java new file mode 100644 index 0000000000000..b67b01d1b38cf --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileEvaluator.java @@ -0,0 +1,152 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.histogram; + +import java.lang.ArithmeticException; +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.ExponentialHistogramBlock; +import org.elasticsearch.compute.data.ExponentialHistogramScratch; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link HistogramPercentile}. + * This class is generated. Edit {@code EvaluatorImplementer} instead. + */ +public final class HistogramPercentileEvaluator implements EvalOperator.ExpressionEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HistogramPercentileEvaluator.class); + + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final EvalOperator.ExpressionEvaluator percentile; + + private final DriverContext driverContext; + + private Warnings warnings; + + public HistogramPercentileEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + EvalOperator.ExpressionEvaluator percentile, DriverContext driverContext) { + this.source = source; + this.value = value; + this.percentile = percentile; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (ExponentialHistogramBlock valueBlock = (ExponentialHistogramBlock) value.eval(page)) { + try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) { + return eval(page.getPositionCount(), valueBlock, percentileBlock); + } + } + } + + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += value.baseRamBytesUsed(); + baseRamBytesUsed += percentile.baseRamBytesUsed(); + return baseRamBytesUsed; + } + + public DoubleBlock eval(int positionCount, ExponentialHistogramBlock valueBlock, + DoubleBlock percentileBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + ExponentialHistogramScratch valueScratch = new ExponentialHistogramScratch(); + position: for (int p = 0; p < positionCount; p++) { + switch (valueBlock.getValueCount(p)) { + case 0: + result.appendNull(); + continue position; + case 1: + break; + default: + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + result.appendNull(); + continue position; + } + switch (percentileBlock.getValueCount(p)) { + case 0: + result.appendNull(); + continue position; + case 1: + break; + default: + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + result.appendNull(); + continue position; + } + ExponentialHistogram value = valueBlock.getExponentialHistogram(valueBlock.getFirstValueIndex(p), valueScratch); + double percentile = percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)); + try { + HistogramPercentile.process(result, value, percentile); + } catch (ArithmeticException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "HistogramPercentileEvaluator[" + "value=" + value + ", percentile=" + percentile + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value, percentile); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final EvalOperator.ExpressionEvaluator.Factory percentile; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + EvalOperator.ExpressionEvaluator.Factory percentile) { + this.source = source; + this.value = value; + this.percentile = percentile; + } + + @Override + public HistogramPercentileEvaluator get(DriverContext context) { + return new HistogramPercentileEvaluator(source, value.get(context), percentile.get(context), context); + } + + @Override + public String toString() { + return "HistogramPercentileEvaluator[" + "value=" + value + ", percentile=" + percentile + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 5087b376b9649..859cad2150685 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.DayName; import org.elasticsearch.xpack.esql.expression.function.scalar.date.MonthName; import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; +import org.elasticsearch.xpack.esql.expression.function.scalar.histogram.HistogramPercentile; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.NetworkDirection; @@ -118,6 +119,7 @@ public static List getNamedWriteables() { entries.add(Tau.ENTRY); entries.add(ToLower.ENTRY); entries.add(ToUpper.ENTRY); + entries.add(HistogramPercentile.ENTRY); entries.addAll(GroupingWritables.getNamedWriteables()); return entries; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentile.java new file mode 100644 index 0000000000000..029c8d065417a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentile.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.histogram; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramQuantile; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; + +/** + * Extracts a percentile value from a single histogram value. + * Note that this function is currently only intended for usage in surrogates and not available as a user-facing function. + * Therefore, it is intentionally not registered in {@link org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry}. + */ +public class HistogramPercentile extends EsqlScalarFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "HistogramPercentile", + HistogramPercentile::new + ); + + private final Expression histogram; + private final Expression percentile; + + @FunctionInfo(returnType = { "double" }) + public HistogramPercentile( + Source source, + @Param(name = "histogram", type = { "exponential_histogram" }) Expression histogram, + @Param(name = "percentile", type = { "double", "integer", "long", "unsigned_long" }) Expression percentile + ) { + super(source, List.of(histogram, percentile)); + this.histogram = histogram; + this.percentile = percentile; + } + + private HistogramPercentile(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + Expression histogram() { + return histogram; + } + + Expression percentile() { + return percentile; + } + + @Override + protected TypeResolution resolveType() { + return isType(histogram, dt -> dt == DataType.EXPONENTIAL_HISTOGRAM, sourceText(), DEFAULT, "exponential_histogram").and( + isType(percentile, DataType::isNumeric, sourceText(), DEFAULT, "numeric types") + ); + } + + @Override + public DataType dataType() { + return DataType.DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new HistogramPercentile(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public boolean foldable() { + return histogram.foldable() && percentile.foldable(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, HistogramPercentile::new, histogram, percentile); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(histogram); + out.writeNamedWriteable(percentile); + } + + @Evaluator(warnExceptions = ArithmeticException.class) + static void process(DoubleBlock.Builder resultBuilder, ExponentialHistogram value, double percentile) { + if (percentile < 0.0 || percentile > 100.0) { + throw new ArithmeticException("Percentile value must be in the range [0, 100], got: " + percentile); + } + double result = ExponentialHistogramQuantile.getQuantile(value, percentile / 100.0); + if (Double.isNaN(result)) { // can happen if the histogram is empty + resultBuilder.appendNull(); + } else { + resultBuilder.appendDouble(result); + } + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var fieldEvaluator = toEvaluator.apply(histogram); + var percentileEvaluator = Cast.cast(source(), percentile.dataType(), DataType.DOUBLE, toEvaluator.apply(percentile)); + return new HistogramPercentileEvaluator.Factory(source(), fieldEvaluator, percentileEvaluator); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 04c6811619ca0..0f735f68317ec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1509,6 +1509,19 @@ public static List aggregateMetricDoubleCases() { ); } + /** + * Generate cases for {@link DataType#EXPONENTIAL_HISTOGRAM}. + */ + public static List exponentialHistogramCases() { + return List.of( + new TypedDataSupplier( + "", + EsqlTestUtils::randomExponentialHistogram, + DataType.EXPONENTIAL_HISTOGRAM + ) + ); + } + public static String getCastEvaluator(String original, DataType current, DataType target) { if (current == target) { return original; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileErrorTests.java new file mode 100644 index 0000000000000..a199c058e3b59 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.histogram; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class HistogramPercentileErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(HistogramPercentileTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new HistogramPercentile(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "exponential_histogram"; + case 1 -> "numeric types"; + default -> throw new IllegalArgumentException("Unexpected parameter position: " + p); + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileSerializationTests.java new file mode 100644 index 0000000000000..56e187dea2f41 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.histogram; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class HistogramPercentileSerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected HistogramPercentile createTestInstance() { + return new HistogramPercentile(randomSource(), randomChild(), randomChild()); + } + + @Override + protected HistogramPercentile mutateInstance(HistogramPercentile instance) throws IOException { + return new HistogramPercentile( + randomSource(), + randomValueOtherThan(instance.histogram(), AbstractExpressionSerializationTests::randomChild), + randomValueOtherThan(instance.percentile(), AbstractExpressionSerializationTests::randomChild) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileTests.java new file mode 100644 index 0000000000000..1fe08e4b95d77 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/HistogramPercentileTests.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.histogram; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramQuantile; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.getCastEvaluator; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.getSuppliersForNumericType; +import static org.hamcrest.Matchers.equalTo; + +public class HistogramPercentileTests extends AbstractScalarFunctionTestCase { + + public HistogramPercentileTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + + List validPercentileSuppliers = Stream.of( + DataType.DOUBLE, + DataType.INTEGER, + DataType.LONG, + DataType.UNSIGNED_LONG + ).filter(DataType::isNumeric).flatMap(type -> getSuppliersForNumericType(type, 0.0, 100.0, true).stream()).toList(); + + List invalidPercentileValues = List.of(-0.01, 100.05, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + + List invalidPercentileSuppliers = invalidPercentileValues.stream() + .map(value -> new TestCaseSupplier.TypedDataSupplier("<" + value + " double>", () -> value, DataType.DOUBLE)) + .toList(); + + List allPercentiles = Stream.concat( + validPercentileSuppliers.stream(), + invalidPercentileSuppliers.stream() + ).toList(); + + TestCaseSupplier.casesCrossProduct((histogramObj, percentileObj) -> { + ExponentialHistogram histogram = (ExponentialHistogram) histogramObj; + Number percentile = (Number) percentileObj; + double percVal = percentile.doubleValue(); + if (percVal < 0 || percVal > 100 || Double.isNaN(percVal)) { + return null; + } + double result = ExponentialHistogramQuantile.getQuantile(histogram, percVal / 100.0); + return Double.isNaN(result) ? null : result; + }, + TestCaseSupplier.exponentialHistogramCases(), + allPercentiles, + (histoType, percentileType) -> equalTo( + "HistogramPercentileEvaluator[value=Attribute[channel=0], percentile=" + + getCastEvaluator("Attribute[channel=1]", percentileType, DataType.DOUBLE) + + "]" + ), + (typedHistoData, typedPercentileData) -> { + Object percentile = typedPercentileData.getValue(); + if (invalidPercentileValues.contains(percentile)) { + return List.of( + "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.", + "Line 1:1: java.lang.ArithmeticException: Percentile value must be in the range [0, 100], got: " + percentile + ); + } else { + return List.of(); + } + }, + suppliers, + DataType.DOUBLE, + false + ); + + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new HistogramPercentile(source, args.get(0), args.get(1)); + } + +}