From d839a7acada0152e53b8b12cf6df65d6a7000ed3 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 2 Mar 2026 17:16:42 +0100 Subject: [PATCH 1/2] Reduce LuceneOperator.Status memory consumption with large QueryDSL queries (#143175) --- docs/changelog/143175.yaml | 6 +++++ .../compute/lucene/LuceneOperator.java | 23 +++++++++++++++---- .../esql/qa/single_node/PushQueriesIT.java | 14 +++++++---- 3 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/143175.yaml diff --git a/docs/changelog/143175.yaml b/docs/changelog/143175.yaml new file mode 100644 index 0000000000000..56e8c93bf11aa --- /dev/null +++ b/docs/changelog/143175.yaml @@ -0,0 +1,6 @@ +area: ES|QL +issues: + - 143164 +pr: 143175 +summary: Reduce `LuceneOperator.Status` memory consumption with large QueryDSL queries +type: bug diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java index 946ab2fca94d1..efd9c3fcee664 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java @@ -57,8 +57,8 @@ public abstract class LuceneOperator extends SourceOperator { final int maxPageSize; private final LuceneSliceQueue sliceQueue; - final Set processedQueries = new HashSet<>(); - final Set processedShards = new HashSet<>(); + private final Set processedQueries = new TreeSet<>(); + private final Set processedShards = new HashSet<>(); protected LuceneSlice currentSlice; private int sliceIndex; @@ -175,7 +175,7 @@ LuceneScorer getCurrentOrLoadNextScorer() { || currentScorer.weight != currentSlice.weight() // Moved to a new query ) { final Weight weight = currentSlice.weight(); - processedQueries.add(weight.getQuery()); + processedQueries.add(Status.queryString(weight.getQuery())); currentScorer = new LuceneScorer(currentSlice.shardContext(), weight, currentSlice.tags(), leaf); } assert currentScorer.maxPosition <= partialLeaf.maxDoc() : currentScorer.maxPosition + ">" + partialLeaf.maxDoc(); @@ -300,9 +300,24 @@ public static class Status implements Operator.Status { private final long rowsEmitted; private final Map partitioningStrategies; + public static final int QUERY_STRING_TRUNCATION = 500; + + private static String queryString(Query query) { + String queryString = query.toString(); + if (queryString.length() > QUERY_STRING_TRUNCATION) { + return queryString.substring(0, QUERY_STRING_TRUNCATION) + + "...(" + + (queryString.length() - QUERY_STRING_TRUNCATION) + + " more characters[" + + queryString.hashCode() + + "])"; + } + return query.toString(); + } + protected Status(LuceneOperator operator) { processedSlices = operator.processedSlices; - processedQueries = operator.processedQueries.stream().map(Query::toString).collect(Collectors.toCollection(TreeSet::new)); + processedQueries = operator.processedQueries; processNanos = operator.processingNanos; processedShards = new TreeSet<>(operator.processedShards); sliceIndex = operator.sliceIndex; diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 3c792838b014b..b5d4a2ccab2a6 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.compute.lucene.query.LuceneOperator; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; @@ -372,10 +373,7 @@ private void testPushQuery( equalTo(found ? List.of(List.of(value)) : List.of()) ); Matcher luceneQueryMatcher = anyOf( - () -> Iterators.map( - luceneQueryOptions.iterator(), - (String s) -> equalTo(s.replaceAll("%value", value).replaceAll("%different_value", differentValue)) - ) + () -> Iterators.map(luceneQueryOptions.iterator(), (String s) -> queryMatcher(s, value, differentValue)) ); @SuppressWarnings("unchecked") @@ -411,6 +409,14 @@ private void testPushQuery( } } + private Matcher queryMatcher(String queryString, String value, String differentValue) { + queryString = queryString.replaceAll("%value", value).replaceAll("%different_value", differentValue); + if (queryString.length() <= LuceneOperator.Status.QUERY_STRING_TRUNCATION) { + return equalTo(queryString); + } + return startsWith(queryString.substring(0, LuceneOperator.Status.QUERY_STRING_TRUNCATION)); + } + private void indexValue(String value) throws IOException { try { // Delete the index if it has already been created. From 0352b811d6594ee6a5093a248ae8084a78fb8300 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 18 Mar 2026 17:25:29 +0100 Subject: [PATCH 2/2] Backporting to 9.3 required inlining the constant --- .../xpack/esql/qa/single_node/PushQueriesIT.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index b5d4a2ccab2a6..592e3b4fa8eb3 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.compute.lucene.query.LuceneOperator; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; @@ -409,12 +408,17 @@ private void testPushQuery( } } + /** + * Must match {@code LuceneOperator.Status.QUERY_STRING_TRUNCATION} in the compute module. + */ + private static final int QUERY_STRING_TRUNCATION = 500; + private Matcher queryMatcher(String queryString, String value, String differentValue) { queryString = queryString.replaceAll("%value", value).replaceAll("%different_value", differentValue); - if (queryString.length() <= LuceneOperator.Status.QUERY_STRING_TRUNCATION) { + if (queryString.length() <= QUERY_STRING_TRUNCATION) { return equalTo(queryString); } - return startsWith(queryString.substring(0, LuceneOperator.Status.QUERY_STRING_TRUNCATION)); + return startsWith(queryString.substring(0, QUERY_STRING_TRUNCATION)); } private void indexValue(String value) throws IOException {