[9.2] Reduce LuceneOperator.Status memory consumption with large QueryDSL queries (#143175)

docs/changelog/143175.yaml

@@ -0,0 +1,6 @@
+area: ES|QL
+issues:
+ - 143164
+pr: 143175
+summary: Reduce `LuceneOperator.Status` memory consumption with large QueryDSL queries
+type: bug

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java

@@ -58,8 +58,8 @@ public abstract class LuceneOperator extends SourceOperator {
     final int maxPageSize;
     private final LuceneSliceQueue sliceQueue;
 
-    final Set<Query> processedQueries = new HashSet<>();
-    final Set<String> processedShards = new HashSet<>();
+    private final Set<String> processedQueries = new TreeSet<>();
+    private final Set<String> processedShards = new HashSet<>();
 
     protected LuceneSlice currentSlice;
     private int sliceIndex;
@@ -184,7 +184,7 @@ LuceneScorer getCurrentOrLoadNextScorer() {
                 || currentScorer.weight != currentSlice.weight() // Moved to a new query
             ) {
                 final Weight weight = currentSlice.weight();
-                processedQueries.add(weight.getQuery());
+                processedQueries.add(Status.queryString(weight.getQuery()));
                 currentScorer = new LuceneScorer(currentSlice.shardContext(), weight, currentSlice.tags(), leaf);
             }
             assert currentScorer.maxPosition <= partialLeaf.maxDoc() : currentScorer.maxPosition + ">" + partialLeaf.maxDoc();
@@ -316,9 +316,24 @@ public static class Status implements Operator.Status {
         private final long rowsEmitted;
         private final Map<String, LuceneSliceQueue.PartitioningStrategy> partitioningStrategies;
 
+        public static final int QUERY_STRING_TRUNCATION = 500;
+
+        private static String queryString(Query query) {
+            String queryString = query.toString();
+            if (queryString.length() > QUERY_STRING_TRUNCATION) {
+                return queryString.substring(0, QUERY_STRING_TRUNCATION)
+                    + "...("
+                    + (queryString.length() - QUERY_STRING_TRUNCATION)
+                    + " more characters["
+                    + queryString.hashCode()
+                    + "])";
+            }
+            return query.toString();
+        }
+
         protected Status(LuceneOperator operator) {
             processedSlices = operator.processedSlices;
-            processedQueries = operator.processedQueries.stream().map(Query::toString).collect(Collectors.toCollection(TreeSet::new));
+            processedQueries = operator.processedQueries;
             processNanos = operator.processingNanos;
             processedShards = new TreeSet<>(operator.processedShards);
             sliceIndex = operator.sliceIndex;

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java

@@ -371,10 +371,7 @@ private void testPushQuery(
             equalTo(found ? List.of(List.of(value)) : List.of())
         );
         Matcher<String> luceneQueryMatcher = anyOf(
-            () -> Iterators.map(
-                luceneQueryOptions.iterator(),
-                (String s) -> equalTo(s.replaceAll("%value", value).replaceAll("%different_value", differentValue))
-            )
+            () -> Iterators.map(luceneQueryOptions.iterator(), (String s) -> queryMatcher(s, value, differentValue))
         );
 
         @SuppressWarnings("unchecked")
@@ -410,6 +407,19 @@ private void testPushQuery(
         }
     }
 
+    /**
+     * Must match {@code LuceneOperator.Status.QUERY_STRING_TRUNCATION} in the compute module.
+     */
+    private static final int QUERY_STRING_TRUNCATION = 500;
+
+    private Matcher<String> queryMatcher(String queryString, String value, String differentValue) {
+        queryString = queryString.replaceAll("%value", value).replaceAll("%different_value", differentValue);
+        if (queryString.length() <= QUERY_STRING_TRUNCATION) {
+            return equalTo(queryString);
+        }
+        return startsWith(queryString.substring(0, QUERY_STRING_TRUNCATION));
+    }
+
     private void indexValue(String value) throws IOException {
         try {
             // Delete the index if it has already been created.