From 1c75b41a54ca10292f42e0e92fae1a1d04e22ffe Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 26 Mar 2026 10:54:59 +0200 Subject: [PATCH 1/8] Fix circuit breaker leak in percolator query construction (#144827) (cherry picked from commit b36fdbca3cf487c2c5e1093e3ea930fdfa096a50) # Conflicts: # modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java --- docs/changelog/144827.yaml | 5 + .../percolator/PercolateQueryBuilder.java | 15 ++ .../percolator/QueryBuilderStoreTests.java | 124 ++++++++- .../suggest/phrase/PhraseSuggester.java | 8 +- ...rHitContextBuilderCircuitBreakerTests.java | 137 ++++++++++ .../PhraseSuggesterCircuitBreakerTests.java | 251 ++++++++++++++++++ ...xpressionQueryListCircuitBreakerTests.java | 167 ++++++++++++ 7 files changed, 704 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/144827.yaml create mode 100644 server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java diff --git a/docs/changelog/144827.yaml b/docs/changelog/144827.yaml new file mode 100644 index 0000000000000..61791428d01eb --- /dev/null +++ b/docs/changelog/144827.yaml @@ -0,0 +1,5 @@ +area: Search +issues: [] +pr: 144827 +summary: Fix circuit breaker leak in percolator query construction +type: bug diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 4b1ef8e215e16..daab438b8b8b4 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -694,6 +694,21 @@ public boolean fieldExistsInIndex(String fieldname) { public void addNamedQuery(String name, Query query) { source.addNamedQuery(name, query); } + + @Override + public void addCircuitBreakerMemory(long bytes, String label) { + source.addCircuitBreakerMemory(bytes, label); + } + + @Override + public long getQueryConstructionMemoryUsed() { + return source.getQueryConstructionMemoryUsed(); + } + + @Override + public void releaseQueryConstructionMemory() { + source.releaseQueryConstructionMemory(); + } }; // This means that fields in the query need to exist in the mapping prior to registering this query diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 7e1a8fd74ba93..b29e183d56400 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -20,8 +20,10 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -40,8 +42,12 @@ import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.TestDocumentParserContext; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RegexpQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.script.field.BinaryDocValuesField; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.test.ESTestCase; @@ -55,6 +61,10 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; + +import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; +import static org.hamcrest.Matchers.greaterThan; + public class QueryBuilderStoreTests extends ESTestCase { @Override @@ -117,7 +127,8 @@ public void testStoringQueryBuilders() throws IOException { Collections.emptyList(), IndexMode.STANDARD ); - SearchExecutionContext searchExecutionContext = new SearchExecutionContext( + CircuitBreaker breaker = newLimitedBreaker(ByteSizeValue.ofMb(100)); + SearchExecutionContext baseContext = new SearchExecutionContext( 0, 0, indexSettings, @@ -140,6 +151,7 @@ public void testStoringQueryBuilders() throws IOException { null, MapperMetrics.NOOP ); + SearchExecutionContext searchExecutionContext = new SearchExecutionContext(baseContext, breaker); PercolateQuery.QueryStore queryStore = PercolateQueryBuilder.createStore( fieldMapper.fieldType(), @@ -159,4 +171,114 @@ public void testStoringQueryBuilders() throws IOException { } } } + + public void testCircuitBreakerReleasedAfterPerDocumentQueryConstruction() throws IOException { + CircuitBreaker circuitBreaker = newLimitedBreaker(ByteSizeValue.ofMb(100)); + + String fieldName = "keyword_field"; + QueryBuilder[] queryBuilders = new QueryBuilder[] { + new WildcardQueryBuilder(fieldName, "test*pattern*with*wildcards"), + new RegexpQueryBuilder(fieldName, ".*test.*regexp.*pattern.*"), + new WildcardQueryBuilder(fieldName, "another*wildcard*query"), + new RegexpQueryBuilder(fieldName, "prefix[0-9]+suffix"), }; + + try (Directory directory = newDirectory()) { + IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer()); + config.setMergePolicy(NoMergePolicy.INSTANCE); + BinaryFieldMapper fieldMapper = PercolatorFieldMapper.Builder.createQueryBuilderFieldBuilder( + MapperBuilderContext.root(false, false) + ); + + IndexVersion indexVersion = IndexVersion.current(); + try (IndexWriter indexWriter = new IndexWriter(directory, config)) { + for (QueryBuilder queryBuilder : queryBuilders) { + DocumentParserContext documentParserContext = new TestDocumentParserContext(); + PercolatorFieldMapper.createQueryBuilderField( + indexVersion, + TransportVersion.current(), + fieldMapper, + queryBuilder, + documentParserContext + ); + indexWriter.addDocument(documentParserContext.doc()); + } + } + + NamedWriteableRegistry writeableRegistry = writableRegistry(); + XContentParserConfiguration parserConfig = parserConfig(); + Settings indexSettingsSettings = indexSettings(indexVersion, 1, 1).build(); + IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder("test").settings(indexSettingsSettings).build(), + Settings.EMPTY + ); + + KeywordFieldMapper keywordMapper = new KeywordFieldMapper.Builder(fieldName, indexSettings).build( + MapperBuilderContext.root(false, false) + ); + MappingLookup mappingLookup = MappingLookup.fromMappers( + Mapping.EMPTY, + List.of(keywordMapper), + Collections.emptyList(), + IndexMode.STANDARD + ); + + BytesBinaryIndexFieldData fieldData = new BytesBinaryIndexFieldData( + fieldMapper.fullPath(), + CoreValuesSourceType.KEYWORD, + BinaryDocValuesField::new + ); + BiFunction> indexFieldDataLookup = (mft, fdc) -> fieldData; + + SearchExecutionContext baseContext = new SearchExecutionContext( + 0, + 0, + indexSettings, + null, + indexFieldDataLookup, + null, + mappingLookup, + null, + null, + parserConfig, + writeableRegistry, + null, + null, + System::currentTimeMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + null, + MapperMetrics.NOOP, + SHARD_SEARCH_STATS + ); + SearchExecutionContext searchExecutionContext = new SearchExecutionContext(baseContext, circuitBreaker); + + PercolateQuery.QueryStore queryStore = PercolateQueryBuilder.createStore( + fieldMapper.fieldType(), + false, + searchExecutionContext + ); + + try (IndexReader indexReader = DirectoryReader.open(directory)) { + LeafReaderContext leafContext = indexReader.leaves().get(0); + CheckedFunction queries = queryStore.getQueries(leafContext); + assertEquals(queryBuilders.length, leafContext.reader().numDocs()); + + long baselineUsed = circuitBreaker.getUsed(); + for (int i = 0; i < queryBuilders.length; i++) { + queries.apply(i); + assertThat( + "CB bytes should still be tracked (not leaked) after document " + i, + circuitBreaker.getUsed(), + greaterThan(baselineUsed) + ); + } + + searchExecutionContext.releaseQueryConstructionMemory(); + assertEquals("All CB bytes must be released after the request-end release", baselineUsed, circuitBreaker.getUsed()); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index ebaf969b40aef..8ce7816a9568c 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -135,8 +135,12 @@ public Suggestion> innerExecute( .createParser(searchExecutionContext.getParserConfig(), querySource) ) { QueryBuilder innerQueryBuilder = AbstractQueryBuilder.parseTopLevelQuery(parser); - final ParsedQuery parsedQuery = searchExecutionContext.toQuery(innerQueryBuilder); - collateMatch = Lucene.exists(searcher, parsedQuery.query()); + try { + final ParsedQuery parsedQuery = searchExecutionContext.toQuery(innerQueryBuilder); + collateMatch = Lucene.exists(searcher, parsedQuery.query()); + } finally { + searchExecutionContext.releaseQueryConstructionMemory(); + } } } if (collateMatch == false && collatePrune == false) { diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java new file mode 100644 index 0000000000000..95548d856cb57 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.index.query; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.fetch.subphase.InnerHitsContext; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class InnerHitContextBuilderCircuitBreakerTests extends ESTestCase { + + @Override + protected NamedWriteableRegistry writableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + public void testCBTrackedDuringInnerHitsAndReleasedAtRequestEnd() throws IOException { + Directory dir = new ByteBuffersDirectory(); + // Empty index – we just need a valid IndexSearcher for the context. + try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new WhitespaceAnalyzer()))) { + // intentionally empty + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + IndexSearcher searcher = new IndexSearcher(reader); + + IndexVersion indexVersion = IndexVersion.current(); + Settings indexSettingsSettings = indexSettings(indexVersion, 1, 1).build(); + IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder("test").settings(indexSettingsSettings).build(), + Settings.EMPTY + ); + KeywordFieldMapper fieldMapper = new KeywordFieldMapper.Builder("field", indexSettings).build( + MapperBuilderContext.root(false, false) + ); + MappingLookup mappingLookup = MappingLookup.fromMappers( + Mapping.EMPTY, + List.of(fieldMapper), + Collections.emptyList(), + IndexMode.STANDARD + ); + + SearchExecutionContext baseCtx = new SearchExecutionContext( + 0, + 0, + indexSettings, + null, + null, + null, + mappingLookup, + null, + null, + parserConfig(), + writableRegistry(), + null, + searcher, + System::currentTimeMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + null, + MapperMetrics.NOOP, + SHARD_SEARCH_STATS + ); + + CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); + SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); + QueryBuilder innerQuery = new WildcardQueryBuilder("field", "*test*pattern*"); + InnerHitBuilder innerHitBuilder = new InnerHitBuilder("test_inner"); + InnerHitContextBuilder builder = new InnerHitContextBuilder(innerQuery, innerHitBuilder, Collections.emptyMap()) { + @Override + protected void doBuild(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) {} + }; + + InnerHitsContext.InnerHitSubContext subContext = org.mockito.Mockito.mock(InnerHitsContext.InnerHitSubContext.class); + + long baselineUsed = cb.getUsed(); + + int iterations = 5; + for (int i = 0; i < iterations; i++) { + builder.setupInnerHitsContext(ctx, subContext); + assertThat( + "CB bytes must still be tracked (not released early) after iteration " + i, + cb.getUsed(), + greaterThan(baselineUsed) + ); + } + + ctx.releaseQueryConstructionMemory(); + assertThat("All CB bytes must be released after the request-end release", cb.getUsed(), equalTo(baselineUsed)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java b/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java new file mode 100644 index 0000000000000..0977b6c423472 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.search.suggest.phrase; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.shingle.ShingleFilter; +import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.spell.SuggestMode; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRefBuilder; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.TemplateScript; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; +import static org.hamcrest.Matchers.equalTo; + +public class PhraseSuggesterCircuitBreakerTests extends ESTestCase { + + @Override + protected NamedWriteableRegistry writableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + public void testCBReleasedAfterEachCollateIteration() throws IOException { + Directory dir = new ByteBuffersDirectory(); + + Map analyzerMap = new HashMap<>(); + analyzerMap.put("body_ngram", new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new StandardTokenizer(); + ShingleFilter sf = new ShingleFilter(t, 2, 3); + sf.setOutputUnigrams(false); + return new TokenStreamComponents(t, new LowerCaseFilter(sf)); + } + }); + analyzerMap.put("body", new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer t = new StandardTokenizer(); + return new TokenStreamComponents(t, new LowerCaseFilter(t)); + } + }); + PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(), analyzerMap); + + IndexWriterConfig conf = new IndexWriterConfig(wrapper); + try (IndexWriter writer = new IndexWriter(dir, conf)) { + for (String line : new String[] { "captain america", "american ace", "captain marvel", "american hero", "captain planet" }) { + Document doc = new Document(); + doc.add(new Field("body", line, TextField.TYPE_NOT_STORED)); + doc.add(new Field("body_ngram", line, TextField.TYPE_NOT_STORED)); + writer.addDocument(doc); + } + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + IndexSearcher searcher = new IndexSearcher(reader); + + IndexVersion indexVersion = IndexVersion.current(); + Settings indexSettingsSettings = indexSettings(indexVersion, 1, 1).build(); + IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder("test").settings(indexSettingsSettings).build(), + Settings.EMPTY + ); + KeywordFieldMapper bodyMapper = new KeywordFieldMapper.Builder("body", indexSettings).build( + MapperBuilderContext.root(false, false) + ); + MappingLookup mappingLookup = MappingLookup.fromMappers( + Mapping.EMPTY, + List.of(bodyMapper), + Collections.emptyList(), + IndexMode.STANDARD + ); + + SearchExecutionContext baseCtx = new SearchExecutionContext( + 0, + 0, + indexSettings, + null, + null, + null, + mappingLookup, + null, + null, + parserConfig(), + writableRegistry(), + null, + searcher, + System::currentTimeMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + null, + MapperMetrics.NOOP, + SHARD_SEARCH_STATS + ); + CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); + SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); + + TemplateScript.Factory scriptFactory = params -> new TemplateScript(params) { + @Override + public String execute() { + return "{\"wildcard\":{\"body\":{\"value\":\"captain*\"}}}"; + } + }; + + PhraseSuggestionContext suggestion = new PhraseSuggestionContext(ctx); + suggestion.setField("body_ngram"); + suggestion.setAnalyzer(wrapper); + suggestion.setSize(5); + suggestion.setShardSize(5); + suggestion.setGramSize(2); + suggestion.setConfidence(0.0f); + suggestion.setMaxErrors(2.0f); + suggestion.setRequireUnigram(false); + suggestion.setCollateQueryScript(scriptFactory); + suggestion.setText(new BytesRef("captan amrica")); + + PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator(); + generator.setField("body"); + generator.suggestMode(SuggestMode.SUGGEST_MORE_POPULAR); + generator.size(10); + generator.accuracy(0.3f); + generator.minWordLength(2); + suggestion.addGenerator(generator); + + assertEquals("CB must be zero before innerExecute", 0L, cb.getUsed()); + + Suggest.Suggestion> result = + PhraseSuggester.INSTANCE.innerExecute("test", suggestion, searcher, new CharsRefBuilder()); + + assertNotNull("innerExecute must return a result", result); + + assertThat( + "CB tracked bytes must be fully released after innerExecute (fix: release per correction)", + ctx.getQueryConstructionMemoryUsed(), + equalTo(0L) + ); + assertThat("Raw CB usage must be zero after innerExecute", cb.getUsed(), equalTo(0L)); + } + } + + public void testNoCBUsageWithoutCollateScript() throws IOException { + Directory dir = new ByteBuffersDirectory(); + try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new WhitespaceAnalyzer()))) { + Document doc = new Document(); + doc.add(new Field("body", "hello world", TextField.TYPE_NOT_STORED)); + writer.addDocument(doc); + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + IndexSearcher searcher = new IndexSearcher(reader); + CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); + + IndexVersion indexVersion = IndexVersion.current(); + Settings indexSettingsSettings = indexSettings(indexVersion, 1, 1).build(); + IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder("test").settings(indexSettingsSettings).build(), + Settings.EMPTY + ); + SearchExecutionContext baseCtx = new SearchExecutionContext( + 0, + 0, + indexSettings, + null, + null, + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + searcher, + System::currentTimeMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + null, + MapperMetrics.NOOP, + SHARD_SEARCH_STATS + ); + SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); + + PhraseSuggestionContext suggestion = new PhraseSuggestionContext(ctx); + suggestion.setField("body"); + suggestion.setAnalyzer(new WhitespaceAnalyzer()); + suggestion.setSize(5); + suggestion.setShardSize(5); + suggestion.setGramSize(1); + suggestion.setConfidence(0.0f); + suggestion.setText(new BytesRef("hello")); + + PhraseSuggester.INSTANCE.innerExecute("test", suggestion, searcher, new CharsRefBuilder()); + assertThat("No CB bytes should be used when there is no collate script", cb.getUsed(), equalTo(0L)); + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java new file mode 100644 index 0000000000000..3a20ce85a7ec7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.enrich; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; +import static org.hamcrest.Matchers.equalTo; + +public class ExpressionQueryListCircuitBreakerTests extends ESTestCase { + + private static TestThreadPool threadPool; + + @BeforeClass + public static void init() { + threadPool = new TestThreadPool("ExpressionQueryListCircuitBreakerTests"); + } + + @AfterClass + public static void cleanup() throws Exception { + terminate(threadPool); + threadPool = null; + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + @Override + protected NamedWriteableRegistry writableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + public void testCBReleasedAfterEachGetQueryCall() throws IOException { + Directory dir = new ByteBuffersDirectory(); + // Empty index – we only need a valid IndexSearcher for the context. + try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new WhitespaceAnalyzer()))) { + // intentionally empty + } + + var registeredSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + registeredSettings.addAll(EsqlFlags.ALL_ESQL_FLAGS_SETTINGS); + + try ( + DirectoryReader reader = DirectoryReader.open(dir); + var clusterService = ClusterServiceUtils.createClusterService( + threadPool, + new ClusterSettings(Settings.EMPTY, registeredSettings) + ) + ) { + + IndexSearcher searcher = new IndexSearcher(reader); + IndexVersion indexVersion = IndexVersion.current(); + Settings indexSettingsSettings = indexSettings(indexVersion, 1, 1).build(); + IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder("test").settings(indexSettingsSettings).build(), + Settings.EMPTY + ); + KeywordFieldMapper fieldMapper = new KeywordFieldMapper.Builder("field", indexSettings).build( + MapperBuilderContext.root(false, false) + ); + MappingLookup mappingLookup = MappingLookup.fromMappers( + Mapping.EMPTY, + List.of(fieldMapper), + Collections.emptyList(), + IndexMode.STANDARD + ); + + SearchExecutionContext baseCtx = new SearchExecutionContext( + 0, + 0, + indexSettings, + null, + null, + null, + mappingLookup, + null, + null, + parserConfig(), + writableRegistry(), + null, + searcher, + System::currentTimeMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + null, + MapperMetrics.NOOP, + SHARD_SEARCH_STATS + ); + + CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); + SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); + WildcardQueryBuilder pushedQuery = new WildcardQueryBuilder("field", "*test*pattern*"); + ExpressionQueryList queryList = ExpressionQueryList.fieldBasedJoin( + Collections.emptyList(), + ctx, + null, + pushedQuery, + clusterService, + AliasFilter.EMPTY + ); + + assertEquals("CB must be zero before any getQuery() call", 0L, cb.getUsed()); + + int positions = 10; + for (int i = 0; i < positions; i++) { + long cbBefore = cb.getUsed(); + + queryList.getQuery(i, null, ctx); + + assertThat( + "CB tracked bytes must be zero after getQuery() (position " + i + ")", + ctx.getQueryConstructionMemoryUsed(), + equalTo(0L) + ); + assertThat("Raw CB usage must return to baseline after getQuery() (position " + i + ")", cb.getUsed(), equalTo(cbBefore)); + } + assertThat("No CB bytes should remain after all positions", cb.getUsed(), equalTo(0L)); + } + } +} From 2abff8e5eec15b2acefc15e27290e5b0d1b4e22f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 26 Mar 2026 12:14:26 +0000 Subject: [PATCH 2/8] [CI] Auto commit changes from spotless --- .../org/elasticsearch/percolator/QueryBuilderStoreTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index b29e183d56400..c7c39f8ab354b 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -61,7 +61,6 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; - import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; import static org.hamcrest.Matchers.greaterThan; From 53f3084cac9c2a159e9018ec5b9d1eac53a8c42c Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 26 Mar 2026 17:46:50 +0200 Subject: [PATCH 3/8] Delete x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java --- ...xpressionQueryListCircuitBreakerTests.java | 167 ------------------ 1 file changed, 167 deletions(-) delete mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java deleted file mode 100644 index 3a20ce85a7ec7..0000000000000 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryListCircuitBreakerTests.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.enrich; - -import org.apache.lucene.analysis.core.WhitespaceAnalyzer; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.MapperMetrics; -import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.MappingLookup; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.index.query.WildcardQueryBuilder; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.internal.AliasFilter; -import org.elasticsearch.test.ClusterServiceUtils; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.esql.plugin.EsqlFlags; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; - -import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; -import static org.hamcrest.Matchers.equalTo; - -public class ExpressionQueryListCircuitBreakerTests extends ESTestCase { - - private static TestThreadPool threadPool; - - @BeforeClass - public static void init() { - threadPool = new TestThreadPool("ExpressionQueryListCircuitBreakerTests"); - } - - @AfterClass - public static void cleanup() throws Exception { - terminate(threadPool); - threadPool = null; - } - - @Override - protected NamedXContentRegistry xContentRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - return new NamedXContentRegistry(searchModule.getNamedXContents()); - } - - @Override - protected NamedWriteableRegistry writableRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - return new NamedWriteableRegistry(searchModule.getNamedWriteables()); - } - - public void testCBReleasedAfterEachGetQueryCall() throws IOException { - Directory dir = new ByteBuffersDirectory(); - // Empty index – we only need a valid IndexSearcher for the context. - try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new WhitespaceAnalyzer()))) { - // intentionally empty - } - - var registeredSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - registeredSettings.addAll(EsqlFlags.ALL_ESQL_FLAGS_SETTINGS); - - try ( - DirectoryReader reader = DirectoryReader.open(dir); - var clusterService = ClusterServiceUtils.createClusterService( - threadPool, - new ClusterSettings(Settings.EMPTY, registeredSettings) - ) - ) { - - IndexSearcher searcher = new IndexSearcher(reader); - IndexVersion indexVersion = IndexVersion.current(); - Settings indexSettingsSettings = indexSettings(indexVersion, 1, 1).build(); - IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder("test").settings(indexSettingsSettings).build(), - Settings.EMPTY - ); - KeywordFieldMapper fieldMapper = new KeywordFieldMapper.Builder("field", indexSettings).build( - MapperBuilderContext.root(false, false) - ); - MappingLookup mappingLookup = MappingLookup.fromMappers( - Mapping.EMPTY, - List.of(fieldMapper), - Collections.emptyList(), - IndexMode.STANDARD - ); - - SearchExecutionContext baseCtx = new SearchExecutionContext( - 0, - 0, - indexSettings, - null, - null, - null, - mappingLookup, - null, - null, - parserConfig(), - writableRegistry(), - null, - searcher, - System::currentTimeMillis, - null, - null, - () -> true, - null, - Collections.emptyMap(), - null, - MapperMetrics.NOOP, - SHARD_SEARCH_STATS - ); - - CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); - SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); - WildcardQueryBuilder pushedQuery = new WildcardQueryBuilder("field", "*test*pattern*"); - ExpressionQueryList queryList = ExpressionQueryList.fieldBasedJoin( - Collections.emptyList(), - ctx, - null, - pushedQuery, - clusterService, - AliasFilter.EMPTY - ); - - assertEquals("CB must be zero before any getQuery() call", 0L, cb.getUsed()); - - int positions = 10; - for (int i = 0; i < positions; i++) { - long cbBefore = cb.getUsed(); - - queryList.getQuery(i, null, ctx); - - assertThat( - "CB tracked bytes must be zero after getQuery() (position " + i + ")", - ctx.getQueryConstructionMemoryUsed(), - equalTo(0L) - ); - assertThat("Raw CB usage must return to baseline after getQuery() (position " + i + ")", cb.getUsed(), equalTo(cbBefore)); - } - assertThat("No CB bytes should remain after all positions", cb.getUsed(), equalTo(0L)); - } - } -} From c0e31a0cbc42e3988b8bdc526067d7b9078cef1c Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 26 Mar 2026 17:47:02 +0200 Subject: [PATCH 4/8] update --- .claude/worktrees/affectionate-moore | 1 + .claude/worktrees/blissful-ride | 1 + .claude/worktrees/fervent-kare | 1 + .../elasticsearch/percolator/QueryBuilderStoreTests.java | 4 +--- .../query/InnerHitContextBuilderCircuitBreakerTests.java | 6 ++---- .../phrase/PhraseSuggesterCircuitBreakerTests.java | 9 +++------ 6 files changed, 9 insertions(+), 13 deletions(-) create mode 160000 .claude/worktrees/affectionate-moore create mode 160000 .claude/worktrees/blissful-ride create mode 160000 .claude/worktrees/fervent-kare diff --git a/.claude/worktrees/affectionate-moore b/.claude/worktrees/affectionate-moore new file mode 160000 index 0000000000000..cacbbd4e18542 --- /dev/null +++ b/.claude/worktrees/affectionate-moore @@ -0,0 +1 @@ +Subproject commit cacbbd4e18542ddbe02aa6245d5c1087b989bbd9 diff --git a/.claude/worktrees/blissful-ride b/.claude/worktrees/blissful-ride new file mode 160000 index 0000000000000..cacbbd4e18542 --- /dev/null +++ b/.claude/worktrees/blissful-ride @@ -0,0 +1 @@ +Subproject commit cacbbd4e18542ddbe02aa6245d5c1087b989bbd9 diff --git a/.claude/worktrees/fervent-kare b/.claude/worktrees/fervent-kare new file mode 160000 index 0000000000000..453425a8c6627 --- /dev/null +++ b/.claude/worktrees/fervent-kare @@ -0,0 +1 @@ +Subproject commit 453425a8c66272cbbc65a67d638128b3621aebfd diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index c7c39f8ab354b..2276ee046e674 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -61,7 +61,6 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; -import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; import static org.hamcrest.Matchers.greaterThan; public class QueryBuilderStoreTests extends ESTestCase { @@ -249,8 +248,7 @@ public void testCircuitBreakerReleasedAfterPerDocumentQueryConstruction() throws null, Collections.emptyMap(), null, - MapperMetrics.NOOP, - SHARD_SEARCH_STATS + MapperMetrics.NOOP ); SearchExecutionContext searchExecutionContext = new SearchExecutionContext(baseContext, circuitBreaker); diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java index 95548d856cb57..2f8f2f7b4f102 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitContextBuilderCircuitBreakerTests.java @@ -38,7 +38,6 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -72,7 +71,7 @@ public void testCBTrackedDuringInnerHitsAndReleasedAtRequestEnd() throws IOExcep IndexMetadata.builder("test").settings(indexSettingsSettings).build(), Settings.EMPTY ); - KeywordFieldMapper fieldMapper = new KeywordFieldMapper.Builder("field", indexSettings).build( + KeywordFieldMapper fieldMapper = new KeywordFieldMapper.Builder("field", indexSettings.getIndexVersionCreated()).build( MapperBuilderContext.root(false, false) ); MappingLookup mappingLookup = MappingLookup.fromMappers( @@ -103,8 +102,7 @@ public void testCBTrackedDuringInnerHitsAndReleasedAtRequestEnd() throws IOExcep null, Collections.emptyMap(), null, - MapperMetrics.NOOP, - SHARD_SEARCH_STATS + MapperMetrics.NOOP ); CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); diff --git a/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java b/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java index 0977b6c423472..1c084f7a65870 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggesterCircuitBreakerTests.java @@ -53,7 +53,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.index.query.SearchExecutionContextHelper.SHARD_SEARCH_STATS; import static org.hamcrest.Matchers.equalTo; public class PhraseSuggesterCircuitBreakerTests extends ESTestCase { @@ -111,7 +110,7 @@ protected TokenStreamComponents createComponents(String fieldName) { IndexMetadata.builder("test").settings(indexSettingsSettings).build(), Settings.EMPTY ); - KeywordFieldMapper bodyMapper = new KeywordFieldMapper.Builder("body", indexSettings).build( + KeywordFieldMapper bodyMapper = new KeywordFieldMapper.Builder("body", indexSettings.getIndexVersionCreated()).build( MapperBuilderContext.root(false, false) ); MappingLookup mappingLookup = MappingLookup.fromMappers( @@ -142,8 +141,7 @@ protected TokenStreamComponents createComponents(String fieldName) { null, Collections.emptyMap(), null, - MapperMetrics.NOOP, - SHARD_SEARCH_STATS + MapperMetrics.NOOP ); CircuitBreaker cb = newLimitedBreaker(ByteSizeValue.ofMb(100)); SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); @@ -230,8 +228,7 @@ public void testNoCBUsageWithoutCollateScript() throws IOException { null, Collections.emptyMap(), null, - MapperMetrics.NOOP, - SHARD_SEARCH_STATS + MapperMetrics.NOOP ); SearchExecutionContext ctx = new SearchExecutionContext(baseCtx, cb); From ecfb90d9064e42924d784437a6c5b5899f0fe295 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 26 Mar 2026 17:51:58 +0200 Subject: [PATCH 5/8] Remove accidental .claude files --- .claude/CLAUDE.md | 303 --------------------------- .claude/settings.local.json | 23 -- .claude/worktrees/affectionate-moore | 1 - .claude/worktrees/blissful-ride | 1 - .claude/worktrees/fervent-kare | 1 - 5 files changed, 329 deletions(-) delete mode 100644 .claude/CLAUDE.md delete mode 100644 .claude/settings.local.json delete mode 160000 .claude/worktrees/affectionate-moore delete mode 160000 .claude/worktrees/blissful-ride delete mode 160000 .claude/worktrees/fervent-kare diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md deleted file mode 100644 index 6829477ecaa41..0000000000000 --- a/.claude/CLAUDE.md +++ /dev/null @@ -1,303 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -Elasticsearch is a distributed search and analytics engine built with Gradle on Java. The codebase consists of a minimal core with functionality delivered through a plugin/module architecture, separating open-source (OSS) and commercial (X-Pack) features under different licenses. - -## Build Commands - -### Building - -```bash -# Build a distribution for your local OS -./gradlew localDistro - -# Build platform-specific distributions -./gradlew :distribution:archives:linux-tar:assemble -./gradlew :distribution:archives:darwin-tar:assemble -./gradlew :distribution:archives:darwin-aarch64-tar:assemble -./gradlew :distribution:archives:windows-zip:assemble - -# Build Docker images -./gradlew buildDockerImage -./gradlew buildAarch64DockerImage -``` - -### Running - -```bash -# Run Elasticsearch from source -./gradlew run - -# Run with remote debugging (port 5005) -./gradlew run --debug-jvm - -# Debug the CLI launcher (port 5107+) -./gradlew run --debug-cli-jvm - -# Run with different distribution (default is full, not OSS) -./gradlew run -Drun.distribution=oss -``` - -### Testing - -```bash -# Run all tests for a specific project -./gradlew :server:test - -# Run a single test class -./gradlew :server:test --tests org.elasticsearch.cluster.SnapshotsInProgressTests - -# Run a single test method -./gradlew :server:test --tests "org.elasticsearch.cluster.SnapshotsInProgressTests.testClone" - -# Run with specific seed for reproducibility -./gradlew :server:test --tests "..." -Dtests.seed= - -# Run integration tests -./gradlew :server:internalClusterTest - -# Run REST API tests -./gradlew :rest-api-spec:yamlRestTest - -# Run all precommit checks (formatting, license headers, etc.) -./gradlew precommit - -# Check for forbidden API usage -./gradlew forbiddenApis - -# Run spotless to format code -./gradlew spotlessApply -``` - -### Dependency Management - -```bash -# Update dependency verification metadata after adding/updating dependencies -./gradlew --write-verification-metadata sha256 precommit - -# Resolve all dependencies (useful for verification) -./gradlew resolveAllDependencies -``` - -## Architecture Overview - -### Directory Structure - -- **server/** - Core Elasticsearch engine with minimal functionality - - Primary packages: `action`, `cluster`, `discovery`, `index`, `search`, `ingest`, `gateway`, `transport`, `http` - - Contains base `Plugin` class and core Module classes for dependency injection - - Source structure: `src/main/java`, `src/test/java`, `src/internalClusterTest/java`, `src/javaRestTest/java` - -- **modules/** - Built-in modules (32+) that ship with all distributions - - Examples: `ingest-common`, `reindex`, `lang-painless`, `repository-s3`, `analysis-common` - - Always loaded; cannot be disabled - - Cannot contain bin/ directories or packaging files - -- **plugins/** - Optional plugins (18+) that can be installed separately - - Examples: `analysis-icu`, `discovery-ec2`, `repository-hdfs` - - Include example plugins for developers - -- **x-pack/** - Commercial features under Elastic License - - 68+ plugins in `x-pack/plugin/` (security, ml, sql, monitoring, watcher, etc.) - - Core x-pack utilities in `x-pack/plugin/core/` - - Separate licensing and test infrastructure in `x-pack/qa/` - -- **libs/** - Shared libraries (28+) used across the codebase - - `core`, `x-content`, `geo`, `grok`, `plugin-api`, `plugin-analysis-api`, etc. - - Published to Maven with "elasticsearch-" prefix - -- **build-tools/** - Three-tier build system - - `build-conventions/` - Checkstyle and basic conventions - - `build-tools/` - Public build plugins for third-party plugin authors - - `build-tools-internal/` - Internal Elasticsearch build logic - -- **distribution/** - Packaging and distributions - - `archives/` - TAR/ZIP for Linux, macOS, Windows, ARM64 - - `docker/` - Docker image variants (standard, Cloud ESS, Ironbank, Wolfi) - - `packages/` - DEB and RPM packages - - `bwc/` - Backward compatibility testing infrastructure - -- **test/** - Shared test infrastructure - - `framework/` - Core test utilities, base test classes - - `fixtures/` - Test fixtures for external services (AWS, Azure, GCS) - -- **qa/** - Cross-cutting integration tests (34+ suites) - - Upgrade tests: `rolling-upgrade`, `full-cluster-restart`, `mixed-cluster` - - Smoke tests, cross-cluster search tests, packaging tests - -### Plugin System Architecture - -The plugin system is the primary extensibility mechanism: - -1. **Base Plugin Class** - All plugins extend `org.elasticsearch.plugins.Plugin` - -2. **Specialized Plugin Interfaces** - Plugins implement one or more: - - `ActionPlugin` - Add custom REST/transport actions - - `AnalysisPlugin` - Add analyzers, tokenizers, filters - - `ClusterPlugin` - Add cluster-level functionality - - `DiscoveryPlugin` - Add node discovery mechanisms - - `EnginePlugin` - Customize indexing/search engine behavior - - `IndexStorePlugin` - Customize index storage - - `IngestPlugin` - Add ingest processors - - `MapperPlugin` - Add custom field mappers - - `NetworkPlugin` - Customize network layer - - `RepositoryPlugin` - Add snapshot repository types - - `ScriptPlugin` - Add scripting languages - - `SearchPlugin` - Add aggregations, queries, scoring functions - -3. **Module vs Plugin vs X-Pack Plugin**: - - **Modules**: Built-in, always loaded, minimal structure, in `modules/` - - **Plugins**: Optional, installable, can have bin/ files, in `plugins/` - - **X-Pack Plugins**: Commercial, Elastic License, in `x-pack/plugin/` - -4. **Gradle Plugin Declaration** - Use `esplugin` DSL block: - ```gradle - esplugin { - name 'plugin-name' - description 'Plugin description' - classname 'org.elasticsearch.plugin.PluginClass' - } - ``` - -### Dependency Injection Pattern - -Elasticsearch uses custom Module classes (not Spring/Guice) for wiring: -- Server has `*Module.java` classes: `ActionModule`, `ClusterModule`, `SearchModule`, `IndicesModule`, etc. -- Plugins contribute to modules via specialized interfaces -- Components registered in registries: `NamedWriteableRegistry`, `NamedXContentRegistry` - -### Settings System - -- Configuration uses `Setting` classes with validation -- Settings are strongly typed and declared statically -- Settings can be static (require restart) or dynamic (updateable via API) -- Plugin settings must be registered via `Plugin.getSettings()` - -### Build System Details - -**Gradle Composite Build**: -- Three included builds: `build-conventions`, `build-tools`, `build-tools-internal` -- Version catalog in `gradle/build.versions.toml` -- Custom toolchain resolvers for JDK management -- Dependency verification in `gradle/verification-metadata.xml` (required) - -**Custom Gradle Plugins**: -- `elasticsearch.esplugin` - Build Elasticsearch plugins -- `elasticsearch.testclusters` - Set up test clusters -- `elasticsearch.internal-es-plugin` - Internal plugin development -- `elasticsearch.yaml-rest-test` - YAML REST test infrastructure -- `elasticsearch.internal-test-artifact` - Share test fixtures between projects - -**Key Build Concepts**: -- Use task avoidance API: `tasks.register()` not `task` -- Lazy test cluster creation: `testClusters.register()` -- Component metadata rules manage transitive dependencies (see `ComponentMetadataRulesPlugin`) -- All dependency versions managed in `build-tools-internal/version.properties` - -### Test Infrastructure - -**Test Types**: -1. **Unit Tests** - `src/test/java` - Standard JUnit tests with randomized testing framework -2. **Internal Cluster Tests** - `src/internalClusterTest/java` - Multi-node cluster tests in same JVM -3. **Java REST Tests** - `src/javaRestTest/java` - REST API tests using Java client -4. **YAML REST Tests** - `src/yamlRestTest/resources` - Declarative REST tests - -**Test Framework Features**: -- Randomized testing with seeds for reproducibility -- `ESTestCase` base class with utilities -- `@TestLogging` for verbose logging on failures -- Test clusters managed by `testclusters` plugin -- Multiple test types can coexist in same project - -**QA Test Organization**: -- Module/plugin-specific: `/qa/` subdirectories -- Cross-cutting: `qa/` directory at root -- X-Pack: `x-pack/qa/` for commercial feature integration tests - -### Code Organization Patterns - -**Dependency Flow**: `libs/` → `server/` → `modules/` → `plugins/`/`x-pack/` - -**Key Server Packages**: -- `action` - Request/response handling, REST/transport actions -- `cluster` - Cluster state management, master node logic -- `index` - Index-level operations, shards, segments -- `search` - Search request handling, query execution -- `ingest` - Document preprocessing pipeline -- `transport` - Inter-node communication -- `http` - HTTP/REST layer -- `discovery` - Cluster formation and node discovery -- `gateway` - Cluster metadata persistence -- `indices` - Cross-index operations, templates - -**Common Patterns**: -- Immutable cluster state with copy-on-write updates -- Async operations with `ActionListener` callbacks -- Streaming serialization with `StreamInput`/`StreamOutput` -- XContent (JSON/YAML) parsing with `XContentParser` -- Thread pools for different operation types - -### Licensing - -- **Server, modules, libs**: Dual-licensed (Server Side Public License v1, Elastic License 2.0, AGPL v3) -- **X-Pack**: Elastic License 2.0 -- License headers required on all source files -- Check with `./gradlew precommit` - -## Development Workflow - -### Code Style - -- Use `./gradlew spotlessApply` to auto-format code -- Follow existing patterns in the codebase -- Do not reformat unchanged lines -- Checkstyle enforced via `precommit` task - -### Making Changes - -1. **Before coding**: Discuss on GitHub issue first -2. **Add tests**: Unit tests required; integration tests for user-facing changes -3. **Run precommit**: `./gradlew precommit` before committing -4. **Run relevant tests**: Test your specific module/plugin -5. **Check forbidden APIs**: Build enforces forbidden API usage rules -6. **Update verification metadata**: If dependencies changed, run with `--write-verification-metadata` - -### Adding Dependencies - -1. Add dependency to appropriate `build.gradle` -2. Add version to `build-tools-internal/version.properties` or `gradle/build.versions.toml` -3. Run `./gradlew --write-verification-metadata sha256 precommit` -4. Manually verify checksums and update `origin` in `gradle/verification-metadata.xml` -5. Add component metadata rules if transitive dependencies need exclusion (see `ComponentMetadataRulesPlugin`) - -### Creating a New Plugin/Module - -1. Create directory in `modules/` or `plugins/` or `x-pack/plugin/` -2. Add `build.gradle` with `esplugin` block -3. Create plugin class extending `Plugin` and implementing specialized interfaces -4. Create `src/main/resources/plugin-descriptor.properties` (generated by gradle) -5. Add tests in `src/test/`, `src/internalClusterTest/`, etc. -6. Plugin is auto-discovered by `settings.gradle` (uses `addSubProjects()`) - -## Common Gotchas - -- **Gradle build fails on deprecation warnings** - Build configured to fail on deprecated API usage -- **Dependency verification failures** - All dependency checksums must be in `gradle/verification-metadata.xml` -- **Test failures with randomization** - Tests use random seeds; use `-Dtests.seed=` to reproduce -- **Module vs Plugin confusion** - Modules are always loaded and simpler; plugins are optional and installable -- **X-Pack requires special build** - Commercial features in x-pack/ have separate licensing -- **Task avoidance** - Always use `tasks.register()` not direct task creation for performance -- **Test source sets** - Different test types require different source sets and gradle plugins -- **Spotless formatting** - CI enforces code formatting; run `spotlessApply` before committing - -## Claude Code Tips - -### Reviewing GitHub PRs -- `gh` CLI may not be authenticated. Use `WebFetch` to fetch PR metadata from `github.com/...` and raw diffs from `patch-diff.githubusercontent.com/raw/...` -- WebFetch summarizes large files. For complete code, fetch individual files via `raw.githubusercontent.com/{org}/{repo}/{sha}/{path}` -- Get the PR's head SHA from the commits page (`/pull/N/commits`) to fetch exact file versions -- Always compare both the old (main) and new (PR head) versions of key files to understand the full diff -- For large PRs, use parallel `WebFetch` and background `Task` agents to fetch files concurrently diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index f2ec2ec0e6256..0000000000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(./gradlew spotlessApply:*)", - "Bash(./gradlew :qa:vector:checkVec:*)", - "Bash(find:*)", - "Bash(./gradlew :server:compileJava:*)", - "Bash(grep:*)", - "Bash(./gradlew :server:test:*)", - "Bash(xargs -I {} sh -c 'echo \"\"=== {} ===\"\" && grep -o \"\"java\\\\.[a-zA-Z]*Exception[^<]*\"\" {} | head -3')", - "Bash(./gradlew:*)", - "Bash(gh pr view:*)", - "Bash(git stash:*)", - "Bash(git checkout:*)", - "WebFetch(domain:github.com)", - "WebFetch(domain:patch-diff.githubusercontent.com)", - "WebFetch(domain:raw.githubusercontent.com)", - "Bash(git log:*)" - ], - "deny": [], - "ask": [] - } -} diff --git a/.claude/worktrees/affectionate-moore b/.claude/worktrees/affectionate-moore deleted file mode 160000 index cacbbd4e18542..0000000000000 --- a/.claude/worktrees/affectionate-moore +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cacbbd4e18542ddbe02aa6245d5c1087b989bbd9 diff --git a/.claude/worktrees/blissful-ride b/.claude/worktrees/blissful-ride deleted file mode 160000 index cacbbd4e18542..0000000000000 --- a/.claude/worktrees/blissful-ride +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cacbbd4e18542ddbe02aa6245d5c1087b989bbd9 diff --git a/.claude/worktrees/fervent-kare b/.claude/worktrees/fervent-kare deleted file mode 160000 index 453425a8c6627..0000000000000 --- a/.claude/worktrees/fervent-kare +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 453425a8c66272cbbc65a67d638128b3621aebfd From a66192a77cad609224dcb009b2acb2813f20b3ca Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Thu, 26 Mar 2026 17:57:12 +0200 Subject: [PATCH 6/8] restore files --- .claude/CLAUDE.md | 303 ++++++++++++++++++++++++++++++++++++ .claude/settings.local.json | 23 +++ 2 files changed, 326 insertions(+) create mode 100644 .claude/CLAUDE.md create mode 100644 .claude/settings.local.json diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md new file mode 100644 index 0000000000000..6829477ecaa41 --- /dev/null +++ b/.claude/CLAUDE.md @@ -0,0 +1,303 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Elasticsearch is a distributed search and analytics engine built with Gradle on Java. The codebase consists of a minimal core with functionality delivered through a plugin/module architecture, separating open-source (OSS) and commercial (X-Pack) features under different licenses. + +## Build Commands + +### Building + +```bash +# Build a distribution for your local OS +./gradlew localDistro + +# Build platform-specific distributions +./gradlew :distribution:archives:linux-tar:assemble +./gradlew :distribution:archives:darwin-tar:assemble +./gradlew :distribution:archives:darwin-aarch64-tar:assemble +./gradlew :distribution:archives:windows-zip:assemble + +# Build Docker images +./gradlew buildDockerImage +./gradlew buildAarch64DockerImage +``` + +### Running + +```bash +# Run Elasticsearch from source +./gradlew run + +# Run with remote debugging (port 5005) +./gradlew run --debug-jvm + +# Debug the CLI launcher (port 5107+) +./gradlew run --debug-cli-jvm + +# Run with different distribution (default is full, not OSS) +./gradlew run -Drun.distribution=oss +``` + +### Testing + +```bash +# Run all tests for a specific project +./gradlew :server:test + +# Run a single test class +./gradlew :server:test --tests org.elasticsearch.cluster.SnapshotsInProgressTests + +# Run a single test method +./gradlew :server:test --tests "org.elasticsearch.cluster.SnapshotsInProgressTests.testClone" + +# Run with specific seed for reproducibility +./gradlew :server:test --tests "..." -Dtests.seed= + +# Run integration tests +./gradlew :server:internalClusterTest + +# Run REST API tests +./gradlew :rest-api-spec:yamlRestTest + +# Run all precommit checks (formatting, license headers, etc.) +./gradlew precommit + +# Check for forbidden API usage +./gradlew forbiddenApis + +# Run spotless to format code +./gradlew spotlessApply +``` + +### Dependency Management + +```bash +# Update dependency verification metadata after adding/updating dependencies +./gradlew --write-verification-metadata sha256 precommit + +# Resolve all dependencies (useful for verification) +./gradlew resolveAllDependencies +``` + +## Architecture Overview + +### Directory Structure + +- **server/** - Core Elasticsearch engine with minimal functionality + - Primary packages: `action`, `cluster`, `discovery`, `index`, `search`, `ingest`, `gateway`, `transport`, `http` + - Contains base `Plugin` class and core Module classes for dependency injection + - Source structure: `src/main/java`, `src/test/java`, `src/internalClusterTest/java`, `src/javaRestTest/java` + +- **modules/** - Built-in modules (32+) that ship with all distributions + - Examples: `ingest-common`, `reindex`, `lang-painless`, `repository-s3`, `analysis-common` + - Always loaded; cannot be disabled + - Cannot contain bin/ directories or packaging files + +- **plugins/** - Optional plugins (18+) that can be installed separately + - Examples: `analysis-icu`, `discovery-ec2`, `repository-hdfs` + - Include example plugins for developers + +- **x-pack/** - Commercial features under Elastic License + - 68+ plugins in `x-pack/plugin/` (security, ml, sql, monitoring, watcher, etc.) + - Core x-pack utilities in `x-pack/plugin/core/` + - Separate licensing and test infrastructure in `x-pack/qa/` + +- **libs/** - Shared libraries (28+) used across the codebase + - `core`, `x-content`, `geo`, `grok`, `plugin-api`, `plugin-analysis-api`, etc. + - Published to Maven with "elasticsearch-" prefix + +- **build-tools/** - Three-tier build system + - `build-conventions/` - Checkstyle and basic conventions + - `build-tools/` - Public build plugins for third-party plugin authors + - `build-tools-internal/` - Internal Elasticsearch build logic + +- **distribution/** - Packaging and distributions + - `archives/` - TAR/ZIP for Linux, macOS, Windows, ARM64 + - `docker/` - Docker image variants (standard, Cloud ESS, Ironbank, Wolfi) + - `packages/` - DEB and RPM packages + - `bwc/` - Backward compatibility testing infrastructure + +- **test/** - Shared test infrastructure + - `framework/` - Core test utilities, base test classes + - `fixtures/` - Test fixtures for external services (AWS, Azure, GCS) + +- **qa/** - Cross-cutting integration tests (34+ suites) + - Upgrade tests: `rolling-upgrade`, `full-cluster-restart`, `mixed-cluster` + - Smoke tests, cross-cluster search tests, packaging tests + +### Plugin System Architecture + +The plugin system is the primary extensibility mechanism: + +1. **Base Plugin Class** - All plugins extend `org.elasticsearch.plugins.Plugin` + +2. **Specialized Plugin Interfaces** - Plugins implement one or more: + - `ActionPlugin` - Add custom REST/transport actions + - `AnalysisPlugin` - Add analyzers, tokenizers, filters + - `ClusterPlugin` - Add cluster-level functionality + - `DiscoveryPlugin` - Add node discovery mechanisms + - `EnginePlugin` - Customize indexing/search engine behavior + - `IndexStorePlugin` - Customize index storage + - `IngestPlugin` - Add ingest processors + - `MapperPlugin` - Add custom field mappers + - `NetworkPlugin` - Customize network layer + - `RepositoryPlugin` - Add snapshot repository types + - `ScriptPlugin` - Add scripting languages + - `SearchPlugin` - Add aggregations, queries, scoring functions + +3. **Module vs Plugin vs X-Pack Plugin**: + - **Modules**: Built-in, always loaded, minimal structure, in `modules/` + - **Plugins**: Optional, installable, can have bin/ files, in `plugins/` + - **X-Pack Plugins**: Commercial, Elastic License, in `x-pack/plugin/` + +4. **Gradle Plugin Declaration** - Use `esplugin` DSL block: + ```gradle + esplugin { + name 'plugin-name' + description 'Plugin description' + classname 'org.elasticsearch.plugin.PluginClass' + } + ``` + +### Dependency Injection Pattern + +Elasticsearch uses custom Module classes (not Spring/Guice) for wiring: +- Server has `*Module.java` classes: `ActionModule`, `ClusterModule`, `SearchModule`, `IndicesModule`, etc. +- Plugins contribute to modules via specialized interfaces +- Components registered in registries: `NamedWriteableRegistry`, `NamedXContentRegistry` + +### Settings System + +- Configuration uses `Setting` classes with validation +- Settings are strongly typed and declared statically +- Settings can be static (require restart) or dynamic (updateable via API) +- Plugin settings must be registered via `Plugin.getSettings()` + +### Build System Details + +**Gradle Composite Build**: +- Three included builds: `build-conventions`, `build-tools`, `build-tools-internal` +- Version catalog in `gradle/build.versions.toml` +- Custom toolchain resolvers for JDK management +- Dependency verification in `gradle/verification-metadata.xml` (required) + +**Custom Gradle Plugins**: +- `elasticsearch.esplugin` - Build Elasticsearch plugins +- `elasticsearch.testclusters` - Set up test clusters +- `elasticsearch.internal-es-plugin` - Internal plugin development +- `elasticsearch.yaml-rest-test` - YAML REST test infrastructure +- `elasticsearch.internal-test-artifact` - Share test fixtures between projects + +**Key Build Concepts**: +- Use task avoidance API: `tasks.register()` not `task` +- Lazy test cluster creation: `testClusters.register()` +- Component metadata rules manage transitive dependencies (see `ComponentMetadataRulesPlugin`) +- All dependency versions managed in `build-tools-internal/version.properties` + +### Test Infrastructure + +**Test Types**: +1. **Unit Tests** - `src/test/java` - Standard JUnit tests with randomized testing framework +2. **Internal Cluster Tests** - `src/internalClusterTest/java` - Multi-node cluster tests in same JVM +3. **Java REST Tests** - `src/javaRestTest/java` - REST API tests using Java client +4. **YAML REST Tests** - `src/yamlRestTest/resources` - Declarative REST tests + +**Test Framework Features**: +- Randomized testing with seeds for reproducibility +- `ESTestCase` base class with utilities +- `@TestLogging` for verbose logging on failures +- Test clusters managed by `testclusters` plugin +- Multiple test types can coexist in same project + +**QA Test Organization**: +- Module/plugin-specific: `/qa/` subdirectories +- Cross-cutting: `qa/` directory at root +- X-Pack: `x-pack/qa/` for commercial feature integration tests + +### Code Organization Patterns + +**Dependency Flow**: `libs/` → `server/` → `modules/` → `plugins/`/`x-pack/` + +**Key Server Packages**: +- `action` - Request/response handling, REST/transport actions +- `cluster` - Cluster state management, master node logic +- `index` - Index-level operations, shards, segments +- `search` - Search request handling, query execution +- `ingest` - Document preprocessing pipeline +- `transport` - Inter-node communication +- `http` - HTTP/REST layer +- `discovery` - Cluster formation and node discovery +- `gateway` - Cluster metadata persistence +- `indices` - Cross-index operations, templates + +**Common Patterns**: +- Immutable cluster state with copy-on-write updates +- Async operations with `ActionListener` callbacks +- Streaming serialization with `StreamInput`/`StreamOutput` +- XContent (JSON/YAML) parsing with `XContentParser` +- Thread pools for different operation types + +### Licensing + +- **Server, modules, libs**: Dual-licensed (Server Side Public License v1, Elastic License 2.0, AGPL v3) +- **X-Pack**: Elastic License 2.0 +- License headers required on all source files +- Check with `./gradlew precommit` + +## Development Workflow + +### Code Style + +- Use `./gradlew spotlessApply` to auto-format code +- Follow existing patterns in the codebase +- Do not reformat unchanged lines +- Checkstyle enforced via `precommit` task + +### Making Changes + +1. **Before coding**: Discuss on GitHub issue first +2. **Add tests**: Unit tests required; integration tests for user-facing changes +3. **Run precommit**: `./gradlew precommit` before committing +4. **Run relevant tests**: Test your specific module/plugin +5. **Check forbidden APIs**: Build enforces forbidden API usage rules +6. **Update verification metadata**: If dependencies changed, run with `--write-verification-metadata` + +### Adding Dependencies + +1. Add dependency to appropriate `build.gradle` +2. Add version to `build-tools-internal/version.properties` or `gradle/build.versions.toml` +3. Run `./gradlew --write-verification-metadata sha256 precommit` +4. Manually verify checksums and update `origin` in `gradle/verification-metadata.xml` +5. Add component metadata rules if transitive dependencies need exclusion (see `ComponentMetadataRulesPlugin`) + +### Creating a New Plugin/Module + +1. Create directory in `modules/` or `plugins/` or `x-pack/plugin/` +2. Add `build.gradle` with `esplugin` block +3. Create plugin class extending `Plugin` and implementing specialized interfaces +4. Create `src/main/resources/plugin-descriptor.properties` (generated by gradle) +5. Add tests in `src/test/`, `src/internalClusterTest/`, etc. +6. Plugin is auto-discovered by `settings.gradle` (uses `addSubProjects()`) + +## Common Gotchas + +- **Gradle build fails on deprecation warnings** - Build configured to fail on deprecated API usage +- **Dependency verification failures** - All dependency checksums must be in `gradle/verification-metadata.xml` +- **Test failures with randomization** - Tests use random seeds; use `-Dtests.seed=` to reproduce +- **Module vs Plugin confusion** - Modules are always loaded and simpler; plugins are optional and installable +- **X-Pack requires special build** - Commercial features in x-pack/ have separate licensing +- **Task avoidance** - Always use `tasks.register()` not direct task creation for performance +- **Test source sets** - Different test types require different source sets and gradle plugins +- **Spotless formatting** - CI enforces code formatting; run `spotlessApply` before committing + +## Claude Code Tips + +### Reviewing GitHub PRs +- `gh` CLI may not be authenticated. Use `WebFetch` to fetch PR metadata from `github.com/...` and raw diffs from `patch-diff.githubusercontent.com/raw/...` +- WebFetch summarizes large files. For complete code, fetch individual files via `raw.githubusercontent.com/{org}/{repo}/{sha}/{path}` +- Get the PR's head SHA from the commits page (`/pull/N/commits`) to fetch exact file versions +- Always compare both the old (main) and new (PR head) versions of key files to understand the full diff +- For large PRs, use parallel `WebFetch` and background `Task` agents to fetch files concurrently diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000000000..f2ec2ec0e6256 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,23 @@ +{ + "permissions": { + "allow": [ + "Bash(./gradlew spotlessApply:*)", + "Bash(./gradlew :qa:vector:checkVec:*)", + "Bash(find:*)", + "Bash(./gradlew :server:compileJava:*)", + "Bash(grep:*)", + "Bash(./gradlew :server:test:*)", + "Bash(xargs -I {} sh -c 'echo \"\"=== {} ===\"\" && grep -o \"\"java\\\\.[a-zA-Z]*Exception[^<]*\"\" {} | head -3')", + "Bash(./gradlew:*)", + "Bash(gh pr view:*)", + "Bash(git stash:*)", + "Bash(git checkout:*)", + "WebFetch(domain:github.com)", + "WebFetch(domain:patch-diff.githubusercontent.com)", + "WebFetch(domain:raw.githubusercontent.com)", + "Bash(git log:*)" + ], + "deny": [], + "ask": [] + } +} From edd0794525a3d9e60228dc86ef1b4236fe60c905 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 27 Mar 2026 09:26:20 +0200 Subject: [PATCH 7/8] update code --- .../elasticsearch/percolator/QueryBuilderStoreTests.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 2276ee046e674..71e6319d584e6 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; -import org.elasticsearch.script.field.BinaryDocValuesField; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.test.ESTestCase; @@ -210,7 +209,7 @@ public void testCircuitBreakerReleasedAfterPerDocumentQueryConstruction() throws Settings.EMPTY ); - KeywordFieldMapper keywordMapper = new KeywordFieldMapper.Builder(fieldName, indexSettings).build( + KeywordFieldMapper keywordMapper = new KeywordFieldMapper.Builder(fieldName, indexSettings.getIndexVersionCreated()).build( MapperBuilderContext.root(false, false) ); MappingLookup mappingLookup = MappingLookup.fromMappers( @@ -222,8 +221,7 @@ public void testCircuitBreakerReleasedAfterPerDocumentQueryConstruction() throws BytesBinaryIndexFieldData fieldData = new BytesBinaryIndexFieldData( fieldMapper.fullPath(), - CoreValuesSourceType.KEYWORD, - BinaryDocValuesField::new + CoreValuesSourceType.KEYWORD ); BiFunction> indexFieldDataLookup = (mft, fdc) -> fieldData; From 50d6d4d26de3399bf586fe3abcbdbc471c9d64e6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 27 Mar 2026 07:37:02 +0000 Subject: [PATCH 8/8] [CI] Auto commit changes from spotless --- .../org/elasticsearch/percolator/QueryBuilderStoreTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 71e6319d584e6..cc646740ecbc0 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -219,10 +219,7 @@ public void testCircuitBreakerReleasedAfterPerDocumentQueryConstruction() throws IndexMode.STANDARD ); - BytesBinaryIndexFieldData fieldData = new BytesBinaryIndexFieldData( - fieldMapper.fullPath(), - CoreValuesSourceType.KEYWORD - ); + BytesBinaryIndexFieldData fieldData = new BytesBinaryIndexFieldData(fieldMapper.fullPath(), CoreValuesSourceType.KEYWORD); BiFunction> indexFieldDataLookup = (mft, fdc) -> fieldData; SearchExecutionContext baseContext = new SearchExecutionContext(