Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ static BytesRef fromGeoGrid(long gridId, @Fixed DataType dataType) {
return new BytesRef(geoGridToString(gridId, dataType));
}

@ConvertEvaluator(extraName = "FromExponentialHistogram")
@ConvertEvaluator(extraName = "FromExponentialHistogram", warnExceptions = { IllegalArgumentException.class })
static BytesRef fromExponentialHistogram(ExponentialHistogram histogram) {
return new BytesRef(exponentialHistogramToString(histogram));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,10 @@ public static String aggregateMetricDoubleBlockToString(AggregateMetricDoubleBlo
}

public static String exponentialHistogramToString(ExponentialHistogram histo) {
int totalBucketCount = histo.negativeBuckets().bucketCount() + histo.positiveBuckets().bucketCount();
if (totalBucketCount >= 100_000) {
throw new IllegalArgumentException("Exponential histogram is too big to be converted to a string");
}
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
ExponentialHistogramXContent.serialize(builder, histo);
return Strings.toString(builder);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,15 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru
return;
}
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
int count = 10_000;
int count;
Set<DataType> complexTypes = Set.of(DataType.EXPONENTIAL_HISTOGRAM);
if (testCase.getData().stream().anyMatch(d -> complexTypes.contains(d.type()))) {
// Limit the amount of data for large types, otherwise the test run very long or even hang
count = 500;
} else {
count = 10_000;
}

int threads = 5;
var evalSupplier = evaluator(expression);
if (testCase.getExpectedBuildEvaluatorWarnings() != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,12 @@
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.compute.data.LongRangeBlockBuilder;
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramBuilder;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.xpack.esql.WriteableExponentialHistogram;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
Expand All @@ -35,6 +39,8 @@
import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TEST_SOURCE;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;

public class ToStringTests extends AbstractConfigurationFunctionTestCase {
public ToStringTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
Expand Down Expand Up @@ -187,13 +193,51 @@ public static Iterable<Object[]> parameters() {
eh -> matchesBytesRef(EsqlDataTypeConverter.exponentialHistogramToString(eh)),
List.of()
);
ExponentialHistogram largeExponentialHistogram = buildDummyHistogram(100_001);
suppliers.add(
new TestCaseSupplier(
"<too many exponential histogram buckets>",
List.of(DataType.EXPONENTIAL_HISTOGRAM),
() -> new TestCaseSupplier.TestCase(
List.of(
new TestCaseSupplier.TypedData(
new WriteableExponentialHistogram(largeExponentialHistogram),
DataType.EXPONENTIAL_HISTOGRAM,
"large exponential histogram"
)
),
"ToStringFromExponentialHistogramEvaluator[histogram=" + read + "]",
DataType.KEYWORD,
is(nullValue())
).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning(
"Line 1:1: java.lang.IllegalArgumentException: Exponential histogram is too big to be converted to a string"
)
)
);

TestCaseSupplier.forUnaryHistogram(
suppliers,
"ToStringFromHistogramEvaluator[histogram=" + read + "]",
DataType.KEYWORD,
h -> matchesBytesRef(EsqlDataTypeConverter.histogramToString(h)),
List.of()
);
// doesn't matter if it's not an actual encoded histogram, as we should never get to the decoding step
BytesRef largeTDigest = new BytesRef(new byte[3 * 1024 * 1024]);
suppliers.add(
new TestCaseSupplier(
"<too large histograms>",
List.of(DataType.HISTOGRAM),
() -> new TestCaseSupplier.TestCase(
List.of(new TestCaseSupplier.TypedData(largeTDigest, DataType.HISTOGRAM, "large histogram")),
"ToStringFromHistogramEvaluator[histogram=" + read + "]",
DataType.KEYWORD,
is(nullValue())
).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.")
.withWarning("Line 1:1: java.lang.IllegalArgumentException: Histogram length is greater than 2MB")
)
);

List<TestCaseSupplier> fixedTimezoneSuppliers = new ArrayList<>();
TestCaseSupplier.forUnaryDateTime(
Expand Down Expand Up @@ -280,6 +324,22 @@ private static List<TestCaseSupplier> casesForDate(String date, String zoneIdStr
);
}

private static ExponentialHistogram buildDummyHistogram(int bucketCount) {
final ExponentialHistogram tooLarge;
try (ExponentialHistogramBuilder builder = ExponentialHistogram.builder((byte) 0, ExponentialHistogramCircuitBreaker.noop())) {
for (int i = 0; i < bucketCount; i++) {
// indices must be unique to count as distinct buckets
if (i % 2 == 0) {
builder.setPositiveBucket(i, 1L);
} else {
builder.setNegativeBucket(i, 1L);
}
}
tooLarge = builder.build();
}
return tooLarge;
}

@Override
protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
return new ToString(source, args.get(0), configuration);
Expand Down