From 7e43341dfb37cb6b6181c3b666cb3c5e14f9a047 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 09:23:02 -0600 Subject: [PATCH 01/19] Add tests which NPE segment has no pattern_text values --- .../PatternTextFieldMapperTests.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 53895e9cea8b7..584ad36da08ca 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -471,6 +471,64 @@ protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + public void testValueFetcherWithMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); + MappedFieldType ft = mapperService.fieldType("field"); + + withLuceneIndex(mapperService, iw -> { + // Segment 1: document with the pattern_text field + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + iw.addDocument(doc1); + iw.commit(); + + // Segment 2: document without the pattern_text field + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); + iw.addDocument(doc2); + iw.commit(); + }, reader -> { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + // Segment with the field should work normally + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("hello world", values.get(0)); + + // Segment without the field should return empty, not throw NPE + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + }); + } + + public void testFieldDataWithMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); + MappedFieldType ft = mapperService.fieldType("field"); + + withLuceneIndex(mapperService, iw -> { + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + iw.addDocument(doc1); + iw.commit(); + + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); + iw.addDocument(doc2); + iw.commit(); + }, reader -> { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + // Segment without the field should not throw NPE + var leafData = fieldData.load(reader.leaves().get(1)); + var bytesValues = leafData.getBytesValues(); + assertFalse(bytesValues.advanceExact(0)); + }); + } + @Override protected List getSortShortcutSupport() { return List.of(); From 23fddb395d073ecdd3c276e805cc0a75e1e30f45 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 09:23:29 -0600 Subject: [PATCH 02/19] Return empty doc values if segment has not pattern_text values --- .../xpack/logsdb/patterntext/PatternTextFieldType.java | 3 ++- .../xpack/logsdb/patterntext/PatternTextIndexFieldData.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index 24ce0055e2dc8..d6c40de3355da 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -138,7 +138,8 @@ public void setNextReader(LeafReaderContext context) { @Override public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { - if (false == docValues.advanceExact(doc)) { + // docValues is null when the segment has no documents with this field + if (docValues == null || false == docValues.advanceExact(doc)) { return List.of(); } return List.of(docValues.binaryValue().utf8ToString()); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java index c9b8cb0c5d1c7..30ae325f91121 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java @@ -87,7 +87,8 @@ public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { @Override public boolean advanceExact(int doc) throws IOException { - return values.advanceExact(doc); + // values is null when the segment has no documents with this field + return values != null && values.advanceExact(doc); } @Override From 068997c895bae0089f9f6259ec4e2228948c0472 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 13:26:16 -0600 Subject: [PATCH 03/19] Add tests for pattern_text with disabled templating Add tests that verify valueFetcher and fieldData return correct values when disable_templating is true. These tests would have caught the NPE on main where PatternTextFallbackDocValues.from() returns null because template_id doc values are never written when templating is disabled. Also strengthen the existing testFieldDataWithMissingFieldSegment to verify the segment with data returns correct values. Co-authored-by: Cursor --- .../PatternTextFieldMapperTests.java | 146 +++++++++++++++++- 1 file changed, 138 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 584ad36da08ca..dded2206c0542 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -476,12 +476,10 @@ public void testValueFetcherWithMissingFieldSegment() throws IOException { MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - // Segment 1: document with the pattern_text field LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); iw.addDocument(doc1); iw.commit(); - // Segment 2: document without the pattern_text field LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); iw.addDocument(doc2); iw.commit(); @@ -491,13 +489,11 @@ public void testValueFetcherWithMissingFieldSegment() throws IOException { SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); ValueFetcher fetcher = ft.valueFetcher(ctx, null); - // Segment with the field should work normally fetcher.setNextReader(reader.leaves().get(0)); List values = fetcher.fetchValues(null, 0, new ArrayList<>()); assertEquals(1, values.size()); assertEquals("hello world", values.get(0)); - // Segment without the field should return empty, not throw NPE fetcher.setNextReader(reader.leaves().get(1)); List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); assertEquals(0, emptyValues.size()); @@ -522,10 +518,144 @@ public void testFieldDataWithMissingFieldSegment() throws IOException { var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); - // Segment without the field should not throw NPE - var leafData = fieldData.load(reader.leaves().get(1)); - var bytesValues = leafData.getBytesValues(); - assertFalse(bytesValues.advanceExact(0)); + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("hello world", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + }); + } + + public void testValueFetcherWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + withLuceneIndex(mapperService, iw -> { + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + iw.addDocument(doc1); + iw.commit(); + + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo bar"))).rootDoc(); + iw.addDocument(doc2); + iw.commit(); + }, reader -> { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values0 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values0.size()); + assertEquals("hello world", values0.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List values1 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values1.size()); + assertEquals("foo bar", values1.get(0)); + }); + } + + public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + withLuceneIndex(mapperService, iw -> { + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + iw.addDocument(doc1); + iw.commit(); + + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); + iw.addDocument(doc2); + iw.commit(); + }, reader -> { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("hello world", values.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + }); + } + + public void testFieldDataWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + withLuceneIndex(mapperService, iw -> { + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + iw.addDocument(doc1); + iw.commit(); + + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); + iw.addDocument(doc2); + iw.commit(); + }, reader -> { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("hello world", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + }); + } + + public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + withLuceneIndex(mapperService, iw -> { + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + iw.addDocument(doc1); + iw.commit(); + + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo bar"))).rootDoc(); + iw.addDocument(doc2); + iw.commit(); + }, reader -> { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("hello world", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertTrue(bytesValues1.advanceExact(0)); + assertEquals("foo bar", bytesValues1.nextValue().utf8ToString()); }); } From 013aae6fa9f754ebe823ad7357fc13fd6eed682c Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 13:26:23 -0600 Subject: [PATCH 04/19] Fix pattern_text value loading with disabled templating When disableTemplating is true (basic license), PatternTextFallbackDocValues.from() always returns null because template_id doc values are never written. This caused an NPE via valueFetcher and returned empty values via fieldData for all pattern_text fields on basic license. Add loadDocValues() to PatternTextFieldType that selects the correct BinaryDocValues source based on the templating and storage flags. Both valueFetcher and PatternTextIndexFieldData now share this single dispatch point, eliminating duplicated logic. Co-authored-by: Cursor --- .../patterntext/PatternTextFieldType.java | 98 ++++++++++++------- .../PatternTextIndexFieldData.java | 6 +- 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index d6c40de3355da..964e3dfa9a157 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -46,7 +46,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -130,7 +129,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public void setNextReader(LeafReaderContext context) { try { - this.docValues = PatternTextFallbackDocValues.from(context.reader(), PatternTextFieldType.this); + this.docValues = loadDocValues(context); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -138,7 +137,6 @@ public void setNextReader(LeafReaderContext context) { @Override public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { - // docValues is null when the segment has no documents with this field if (docValues == null || false == docValues.advanceExact(doc)) { return List.of(); } @@ -147,35 +145,14 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - // PatternedTextCompositeValues may require a stored field, but it handles loading this field internally. return StoredFieldsSpec.NO_REQUIREMENTS; } }; } - private IOFunction, IOException>> getValueFetcherProvider( - SearchExecutionContext searchExecutionContext - ) { - if (disableTemplating) { - return useBinaryDocValuesRawText ? binaryDocValuesFetcher(storedNamed()) : storedFieldFetcher(storedNamed()); - } - - return context -> { - ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null); - valueFetcher.setNextReader(context); - return docID -> { - try { - return valueFetcher.fetchValues(null, docID, new ArrayList<>()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - }; - } - - private static IOFunction, IOException>> binaryDocValuesFetcher(String name) { + private IOFunction, IOException>> getValueFetcherProvider() { return context -> { - var docValues = context.reader().getBinaryDocValues(name); + var docValues = loadDocValues(context); return docId -> { if (docValues != null && docValues.advanceExact(docId)) { return List.of(docValues.binaryValue()); @@ -185,16 +162,59 @@ private static IOFunction, IO }; } - private static IOFunction, IOException>> storedFieldFetcher(String name) { - var loader = StoredFieldLoader.create(false, Set.of(name)); - return context -> { - var leafLoader = loader.getLoader(context, null); - return docId -> { - leafLoader.advanceTo(docId); + BinaryDocValues loadDocValues(LeafReaderContext context) throws IOException { + if (disableTemplating) { + if (useBinaryDocValuesRawText) { + return context.reader().getBinaryDocValues(storedNamed()); + } + return storedFieldAsBinaryDocValues(context, storedNamed()); + } + return PatternTextFallbackDocValues.from(context.reader(), this); + } + + private static BinaryDocValues storedFieldAsBinaryDocValues(LeafReaderContext context, String fieldName) throws IOException { + var loader = StoredFieldLoader.create(false, Set.of(fieldName)); + var leafLoader = loader.getLoader(context, null); + return new BinaryDocValues() { + BytesRef currentValue; + + @Override + public boolean advanceExact(int target) throws IOException { + leafLoader.advanceTo(target); var storedFields = leafLoader.storedFields(); - var values = storedFields.get(name); - return values != null ? values : List.of(); - }; + var fieldValues = storedFields.get(fieldName); + if (fieldValues != null && fieldValues.isEmpty() == false) { + currentValue = (BytesRef) fieldValues.getFirst(); + return true; + } + currentValue = null; + return false; + } + + @Override + public BytesRef binaryValue() { + return currentValue; + } + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return 0; + } }; } @@ -203,12 +223,12 @@ private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext contex if (hasPositions) { return new ConstantScoreQuery(query); } else { - return new ConstantScoreQuery(new SourceConfirmedTextQuery(query, getValueFetcherProvider(context), indexAnalyzer)); + return new ConstantScoreQuery(new SourceConfirmedTextQuery(query, getValueFetcherProvider(), indexAnalyzer)); } } private IntervalsSource toIntervalsSource(IntervalsSource source, Query approximation, SearchExecutionContext searchExecutionContext) { - return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(searchExecutionContext), indexAnalyzer); + return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(), indexAnalyzer); } @Override @@ -381,4 +401,8 @@ boolean useBinaryDocValuesArgs() { return useBinaryDocValuesArgs; } + boolean useBinaryDocValuesRawText() { + return useBinaryDocValuesRawText; + } + } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java index 30ae325f91121..24b15b67cf9d3 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.logsdb.patterntext; -import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; @@ -71,8 +71,7 @@ public LeafFieldData load(LeafReaderContext context) { @Override public LeafFieldData loadDirect(LeafReaderContext context) throws IOException { - LeafReader leafReader = context.reader(); - var values = PatternTextFallbackDocValues.from(leafReader, fieldType); + final BinaryDocValues values = fieldType.loadDocValues(context); return new LeafFieldData() { final ToScriptFieldFactory factory = KeywordDocValuesField::new; @@ -87,7 +86,6 @@ public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { @Override public boolean advanceExact(int doc) throws IOException { - // values is null when the segment has no documents with this field return values != null && values.advanceExact(doc); } From e1cbc1b5ae4f239d186c0fc816a4612cc44d642c Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 13:38:19 -0600 Subject: [PATCH 05/19] Use numbers in test values so will produce pattern_text arg tokens --- .../PatternTextFieldMapperTests.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index dded2206c0542..9a5a34ffd7500 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -476,7 +476,7 @@ public void testValueFetcherWithMissingFieldSegment() throws IOException { MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); iw.addDocument(doc1); iw.commit(); @@ -492,7 +492,7 @@ public void testValueFetcherWithMissingFieldSegment() throws IOException { fetcher.setNextReader(reader.leaves().get(0)); List values = fetcher.fetchValues(null, 0, new ArrayList<>()); assertEquals(1, values.size()); - assertEquals("hello world", values.get(0)); + assertEquals("abc 123", values.get(0)); fetcher.setNextReader(reader.leaves().get(1)); List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); @@ -505,7 +505,7 @@ public void testFieldDataWithMissingFieldSegment() throws IOException { MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); iw.addDocument(doc1); iw.commit(); @@ -521,7 +521,7 @@ public void testFieldDataWithMissingFieldSegment() throws IOException { var leafData0 = fieldData.load(reader.leaves().get(0)); var bytesValues0 = leafData0.getBytesValues(); assertTrue(bytesValues0.advanceExact(0)); - assertEquals("hello world", bytesValues0.nextValue().utf8ToString()); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); var leafData1 = fieldData.load(reader.leaves().get(1)); var bytesValues1 = leafData1.getBytesValues(); @@ -537,11 +537,11 @@ public void testValueFetcherWithDisabledTemplating() throws IOException { MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); iw.addDocument(doc1); iw.commit(); - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo bar"))).rootDoc(); + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc(); iw.addDocument(doc2); iw.commit(); }, reader -> { @@ -553,12 +553,12 @@ public void testValueFetcherWithDisabledTemplating() throws IOException { fetcher.setNextReader(reader.leaves().get(0)); List values0 = fetcher.fetchValues(null, 0, new ArrayList<>()); assertEquals(1, values0.size()); - assertEquals("hello world", values0.get(0)); + assertEquals("abc 123", values0.get(0)); fetcher.setNextReader(reader.leaves().get(1)); List values1 = fetcher.fetchValues(null, 0, new ArrayList<>()); assertEquals(1, values1.size()); - assertEquals("foo bar", values1.get(0)); + assertEquals("foo 12", values1.get(0)); }); } @@ -570,7 +570,7 @@ public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throw MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); iw.addDocument(doc1); iw.commit(); @@ -586,7 +586,7 @@ public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throw fetcher.setNextReader(reader.leaves().get(0)); List values = fetcher.fetchValues(null, 0, new ArrayList<>()); assertEquals(1, values.size()); - assertEquals("hello world", values.get(0)); + assertEquals("abc 123", values.get(0)); fetcher.setNextReader(reader.leaves().get(1)); List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); @@ -602,7 +602,7 @@ public void testFieldDataWithDisabledTemplating() throws IOException { MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); iw.addDocument(doc1); iw.commit(); @@ -618,7 +618,7 @@ public void testFieldDataWithDisabledTemplating() throws IOException { var leafData0 = fieldData.load(reader.leaves().get(0)); var bytesValues0 = leafData0.getBytesValues(); assertTrue(bytesValues0.advanceExact(0)); - assertEquals("hello world", bytesValues0.nextValue().utf8ToString()); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); var leafData1 = fieldData.load(reader.leaves().get(1)); var bytesValues1 = leafData1.getBytesValues(); @@ -634,11 +634,11 @@ public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOExcep MappedFieldType ft = mapperService.fieldType("field"); withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "hello world"))).rootDoc(); + LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); iw.addDocument(doc1); iw.commit(); - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo bar"))).rootDoc(); + LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc(); iw.addDocument(doc2); iw.commit(); }, reader -> { @@ -650,12 +650,12 @@ public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOExcep var leafData0 = fieldData.load(reader.leaves().get(0)); var bytesValues0 = leafData0.getBytesValues(); assertTrue(bytesValues0.advanceExact(0)); - assertEquals("hello world", bytesValues0.nextValue().utf8ToString()); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); var leafData1 = fieldData.load(reader.leaves().get(1)); var bytesValues1 = leafData1.getBytesValues(); assertTrue(bytesValues1.advanceExact(0)); - assertEquals("foo bar", bytesValues1.nextValue().utf8ToString()); + assertEquals("foo 12", bytesValues1.nextValue().utf8ToString()); }); } From 3a0d24ba3683bdc76b7ef5847ce1c113f9dfad20 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 14:20:39 -0600 Subject: [PATCH 06/19] add removed comment --- .../xpack/logsdb/patterntext/PatternTextFieldType.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index 964e3dfa9a157..c1d44a83ecd72 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -145,6 +145,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { + // Pattern Text may require a stored field, but it handles loading this field internally. return StoredFieldsSpec.NO_REQUIREMENTS; } }; From 6f565d827ea94ce75cfa4109e0f5afc8ed637d41 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 20 Feb 2026 15:27:03 -0600 Subject: [PATCH 07/19] Fix pattern_text source-confirmed queries returning BytesRef The value fetcher provider was returning raw BytesRef objects from doc values. SourceIntervalsSource calls value.toString() which on BytesRef produces hex output instead of text, causing intervals queries to never match. Co-authored-by: Cursor --- .../xpack/logsdb/patterntext/PatternTextFieldType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index c1d44a83ecd72..f5a26b205aa29 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -156,7 +156,7 @@ private IOFunction, IOExcepti var docValues = loadDocValues(context); return docId -> { if (docValues != null && docValues.advanceExact(docId)) { - return List.of(docValues.binaryValue()); + return List.of(docValues.binaryValue().utf8ToString()); } return List.of(); }; From 8a70bdf8cfc6107fcc61d09b51e3d94e9504d1df Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 11:19:57 -0600 Subject: [PATCH 08/19] Add intervals query test for disabled templating Verifies that intervals queries match correctly on pattern_text fields with disable_templating=true. --- .../PatternTextFieldMapperTests.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 9a5a34ffd7500..2ff37b3c94f76 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -13,11 +13,14 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.queries.intervals.IntervalQuery; +import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -133,6 +136,27 @@ private void assertPhraseQuery(MapperService mapperService) throws IOException { } } + public void testIntervalsQueryWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "pattern_text").field("disable_templating", true)) + ); + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "the quick brown fox 1"))).rootDoc(); + iw.addDocument(doc); + iw.close(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SearchExecutionContext context = createSearchExecutionContext(mapperService, newSearcher(reader)); + PatternTextFieldType ft = (PatternTextFieldType) mapperService.fieldType("field"); + IntervalsSource intervalsSource = ft.termIntervals(new BytesRef("brown"), context); + Query query = new IntervalQuery("field", intervalsSource); + TopDocs docs = context.searcher().search(query, 1); + assertThat(docs.totalHits.value(), equalTo(1L)); + assertThat(docs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); + } + } + } + @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck( From 9aace571d4d0cb6035e079f28b47f15f49ac86ad Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 11:30:06 -0600 Subject: [PATCH 09/19] move from method to PatternTextDocValues --- .../xpack/logsdb/patterntext/PatternTextDocValues.java | 10 ++++++++++ .../patterntext/PatternTextFallbackDocValues.java | 7 +------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java index 3e98f5866ca3b..67855bfe21926 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java @@ -27,6 +27,16 @@ public final class PatternTextDocValues extends BinaryDocValues { this.argsInfoDocValues = argsInfoDocValues; } + /** + * Creates a {@link PatternTextDocValues} by loading the template, args, and args_info doc values from the given leaf reader. + */ + static PatternTextDocValues from(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException { + SortedSetDocValues templateDocValues = DocValues.getSortedSet(leafReader, fieldType.templateFieldName()); + BinaryDocValues argsDocValues = getArgsDocValues(leafReader, fieldType.argsFieldName(), fieldType.useBinaryDocValuesArgs()); + SortedSetDocValues argsInfoDocValues = DocValues.getSortedSet(leafReader, fieldType.argsInfoFieldName()); + return new PatternTextDocValues(templateDocValues, argsDocValues, argsInfoDocValues); + } + /** * Args columns was originally a SortedSetDocValues column and was replaced with BinaryDocValues. * To maintain backwards compatibility, if a BinaryDocValues column does not exist, use the old SortedSetDocValues. diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java index 1456c44615773..187f7358a892c 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java @@ -21,8 +21,6 @@ import java.util.List; import java.util.Set; -import static org.elasticsearch.xpack.logsdb.patterntext.PatternTextDocValues.getArgsDocValues; - /** * Values which exceed 32kb cannot be stored as sorted set doc values. Such values must be stored outside sorted set doc values. * This class relies on {@link PatternTextDocValues} but can fall back to values that were stored seperately because limit was exceded. @@ -137,10 +135,7 @@ static BinaryDocValues from(LeafReader leafReader, PatternTextFieldType fieldTyp return null; } - SortedSetDocValues templateDocValues = DocValues.getSortedSet(leafReader, fieldType.templateFieldName()); - BinaryDocValues argsDocValues = getArgsDocValues(leafReader, fieldType.argsFieldName(), fieldType.useBinaryDocValuesArgs()); - SortedSetDocValues argsInfoDocValues = DocValues.getSortedSet(leafReader, fieldType.argsInfoFieldName()); - var docValues = new PatternTextDocValues(templateDocValues, argsDocValues, argsInfoDocValues); + var docValues = PatternTextDocValues.from(leafReader, fieldType); FieldInfo fieldInfo = leafReader.getFieldInfos().fieldInfo(fieldType.storedNamed()); if (fieldInfo == null) { From 1298c2eede573c84f2e0f764d66746a17f325a90 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 13:02:20 -0600 Subject: [PATCH 10/19] Centralize doc values loading in PatternTextFallbackDocValues Move loadDocValues and storedFieldAsBinaryDocValues from PatternTextFieldType into PatternTextFallbackDocValues.from(), which now handles dispatch for both templating-enabled and templating-disabled paths. Update BytesRefsFromBinaryBlockLoader to accept LeafReaderContext so blockLoader() can use the unified entry point directly. --- .../BytesRefsFromBinaryBlockLoader.java | 8 +- .../PatternTextFallbackDocValues.java | 72 ++++++++++++++++-- .../patterntext/PatternTextFieldMapper.java | 2 +- .../patterntext/PatternTextFieldType.java | 73 +------------------ .../PatternTextIndexFieldData.java | 2 +- .../PatternTextFieldTypeTests.java | 3 +- 6 files changed, 76 insertions(+), 84 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java index f336764e56664..c88dd69db89bf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java @@ -27,10 +27,10 @@ */ public class BytesRefsFromBinaryBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { - private final IOFunction docValuesSupplier; + private final IOFunction docValuesSupplier; public BytesRefsFromBinaryBlockLoader(String fieldName) { - this(leafReader -> leafReader.getBinaryDocValues(fieldName)); + this(context -> context.reader().getBinaryDocValues(fieldName)); } /** @@ -38,7 +38,7 @@ public BytesRefsFromBinaryBlockLoader(String fieldName) { * This is useful when the doc values are not directly stored in a single field * but are composed of multiple sources, as is the case for Pattern Text. */ - public BytesRefsFromBinaryBlockLoader(IOFunction docValuesSupplier) { + public BytesRefsFromBinaryBlockLoader(IOFunction docValuesSupplier) { this.docValuesSupplier = docValuesSupplier; } @@ -49,7 +49,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { @Override public AllReader reader(LeafReaderContext context) throws IOException { - BinaryDocValues docValues = docValuesSupplier.apply(context.reader()); + BinaryDocValues docValues = docValuesSupplier.apply(context); return createReader(docValues); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java index 187f7358a892c..b957c1215abce 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; @@ -129,7 +130,26 @@ public boolean advanceExact(int target) throws IOException { } } - static BinaryDocValues from(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException { + /** + * Returns doc values for a pattern_text field. If templating is disabled returns an alternative version consisting of either + * a flat binary doc values column or stored fields. The decision between proper pattern_text, flat binary doc values, or + * stored fields occurs at the column level. Returns null if the segment has no values for this field. + */ + static BinaryDocValues from(LeafReaderContext context, PatternTextFieldType fieldType) throws IOException { + if (fieldType.disableTemplating()) { + if (fieldType.useBinaryDocValuesRawText()) { + return context.reader().getBinaryDocValues(fieldType.storedNamed()); + } + return storedFieldAsBinaryDocValues(context, fieldType.storedNamed()); + } + return fromEnabledPatternText(context.reader(), fieldType); + } + + /** + * Returns doc values for a pattern_text field that has templating enabled. Single rows may fall back to binary doc values or a + * stored field. This decision happens on a per-doc basis. Returns null if the segment has no template_id values. + */ + static BinaryDocValues fromEnabledPatternText(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException { SortedSetDocValues templateIdDocValues = DocValues.getSortedSet(leafReader, fieldType.templateIdFieldName()); if (templateIdDocValues.getValueCount() == 0) { return null; @@ -139,20 +159,62 @@ static BinaryDocValues from(LeafReader leafReader, PatternTextFieldType fieldTyp FieldInfo fieldInfo = leafReader.getFieldInfos().fieldInfo(fieldType.storedNamed()); if (fieldInfo == null) { - // If there is no stored subfield (either binary doc values or stored field), - // then there is no need to use PatternTextFallbackDocValues return docValues; } - // load binary doc values (for newer indices that store raw values in binary doc values) if (fieldInfo.getDocValuesType() == DocValuesType.BINARY) { BinaryDocValues rawBinaryDocValues = leafReader.getBinaryDocValues(fieldType.storedNamed()); return new BinaryFallback(docValues, templateIdDocValues, rawBinaryDocValues); } else { - // load stored field loader (for older indices that store raw values in stored fields) StoredFieldLoader storedFieldLoader = StoredFieldLoader.create(false, Set.of(fieldType.storedNamed())); LeafStoredFieldLoader storedTemplateLoader = storedFieldLoader.getLoader(leafReader.getContext(), null); return new LegacyStoredFieldFallback(storedTemplateLoader, fieldType.storedNamed(), docValues, templateIdDocValues); } } + + private static BinaryDocValues storedFieldAsBinaryDocValues(LeafReaderContext context, String fieldName) throws IOException { + var loader = StoredFieldLoader.create(false, Set.of(fieldName)); + var leafLoader = loader.getLoader(context, null); + return new BinaryDocValues() { + BytesRef currentValue; + + @Override + public boolean advanceExact(int target) throws IOException { + leafLoader.advanceTo(target); + var storedFields = leafLoader.storedFields(); + var fieldValues = storedFields.get(fieldName); + if (fieldValues != null && fieldValues.isEmpty() == false) { + currentValue = (BytesRef) fieldValues.getFirst(); + return true; + } + currentValue = null; + return false; + } + + @Override + public BytesRef binaryValue() { + return currentValue; + } + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return 0; + } + }; + } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java index 02e5a604400a2..f59c717971cd3 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java @@ -386,7 +386,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { fullPath(), new PatternTextSyntheticFieldLoaderLayer( fieldType().name(), - leafReader -> PatternTextFallbackDocValues.from(leafReader, fieldType()) + leafReader -> PatternTextFallbackDocValues.fromEnabledPatternText(leafReader, fieldType()) ) ); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index f5a26b205aa29..9edce7b5577db 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -29,9 +29,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.BlockStoredFieldsReader; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -50,7 +48,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; public class PatternTextFieldType extends TextFamilyFieldType { @@ -129,7 +126,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public void setNextReader(LeafReaderContext context) { try { - this.docValues = loadDocValues(context); + this.docValues = PatternTextFallbackDocValues.from(context, PatternTextFieldType.this); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -153,7 +150,7 @@ public StoredFieldsSpec storedFieldsSpec() { private IOFunction, IOException>> getValueFetcherProvider() { return context -> { - var docValues = loadDocValues(context); + var docValues = PatternTextFallbackDocValues.from(context, PatternTextFieldType.this); return docId -> { if (docValues != null && docValues.advanceExact(docId)) { return List.of(docValues.binaryValue().utf8ToString()); @@ -163,62 +160,6 @@ private IOFunction, IOExcepti }; } - BinaryDocValues loadDocValues(LeafReaderContext context) throws IOException { - if (disableTemplating) { - if (useBinaryDocValuesRawText) { - return context.reader().getBinaryDocValues(storedNamed()); - } - return storedFieldAsBinaryDocValues(context, storedNamed()); - } - return PatternTextFallbackDocValues.from(context.reader(), this); - } - - private static BinaryDocValues storedFieldAsBinaryDocValues(LeafReaderContext context, String fieldName) throws IOException { - var loader = StoredFieldLoader.create(false, Set.of(fieldName)); - var leafLoader = loader.getLoader(context, null); - return new BinaryDocValues() { - BytesRef currentValue; - - @Override - public boolean advanceExact(int target) throws IOException { - leafLoader.advanceTo(target); - var storedFields = leafLoader.storedFields(); - var fieldValues = storedFields.get(fieldName); - if (fieldValues != null && fieldValues.isEmpty() == false) { - currentValue = (BytesRef) fieldValues.getFirst(); - return true; - } - currentValue = null; - return false; - } - - @Override - public BytesRef binaryValue() { - return currentValue; - } - - @Override - public int docID() { - throw new UnsupportedOperationException(); - } - - @Override - public int nextDoc() { - throw new UnsupportedOperationException(); - } - - @Override - public int advance(int target) { - throw new UnsupportedOperationException(); - } - - @Override - public long cost() { - return 0; - } - }; - } - private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext context) { // Disable scoring similarly to match_only_text if (hasPositions) { @@ -349,16 +290,6 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (disableTemplating) { - if (useBinaryDocValuesRawText) { - // for newer indices, raw pattern text values are stored in binary doc values - return new BytesRefsFromBinaryBlockLoader(storedNamed()); - } else { - // for older indices (bwc), raw pattern text values are stored in stored fields - return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedNamed()); - } - } - return new BytesRefsFromBinaryBlockLoader(leafReader -> PatternTextFallbackDocValues.from(leafReader, this)); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java index 24b15b67cf9d3..516c2abdb3942 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java @@ -71,7 +71,7 @@ public LeafFieldData load(LeafReaderContext context) { @Override public LeafFieldData loadDirect(LeafReaderContext context) throws IOException { - final BinaryDocValues values = fieldType.loadDocValues(context); + final BinaryDocValues values = PatternTextFallbackDocValues.from(context, fieldType); return new LeafFieldData() { final ToScriptFieldFactory factory = KeywordDocValuesField::new; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java index d74a61e401f4c..57391b13d44f5 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.BlockStoredFieldsReader; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromBinaryBlockLoader; @@ -266,7 +265,7 @@ public void testBlockLoaderWhenTemplatingIsDisabledAndStoredFieldsAreUsed() { false ); BlockLoader blockLoader = ft.blockLoader(null); - assertThat(blockLoader, Matchers.instanceOf(BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader.class)); + assertThat(blockLoader, Matchers.instanceOf(BytesRefsFromBinaryBlockLoader.class)); } } From 90c5199bc81c0c0c700c3ae8e53fdac1ebafa6f7 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 13:21:16 -0600 Subject: [PATCH 11/19] add back some comments --- .../logsdb/patterntext/PatternTextDocValues.java | 10 ---------- .../patterntext/PatternTextFallbackDocValues.java | 11 ++++++++++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java index 67855bfe21926..3e98f5866ca3b 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextDocValues.java @@ -27,16 +27,6 @@ public final class PatternTextDocValues extends BinaryDocValues { this.argsInfoDocValues = argsInfoDocValues; } - /** - * Creates a {@link PatternTextDocValues} by loading the template, args, and args_info doc values from the given leaf reader. - */ - static PatternTextDocValues from(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException { - SortedSetDocValues templateDocValues = DocValues.getSortedSet(leafReader, fieldType.templateFieldName()); - BinaryDocValues argsDocValues = getArgsDocValues(leafReader, fieldType.argsFieldName(), fieldType.useBinaryDocValuesArgs()); - SortedSetDocValues argsInfoDocValues = DocValues.getSortedSet(leafReader, fieldType.argsInfoFieldName()); - return new PatternTextDocValues(templateDocValues, argsDocValues, argsInfoDocValues); - } - /** * Args columns was originally a SortedSetDocValues column and was replaced with BinaryDocValues. * To maintain backwards compatibility, if a BinaryDocValues column does not exist, use the old SortedSetDocValues. diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java index b957c1215abce..6c7c280413325 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.Set; +import static org.elasticsearch.xpack.logsdb.patterntext.PatternTextDocValues.getArgsDocValues; + /** * Values which exceed 32kb cannot be stored as sorted set doc values. Such values must be stored outside sorted set doc values. * This class relies on {@link PatternTextDocValues} but can fall back to values that were stored seperately because limit was exceded. @@ -155,17 +157,24 @@ static BinaryDocValues fromEnabledPatternText(LeafReader leafReader, PatternText return null; } - var docValues = PatternTextDocValues.from(leafReader, fieldType); + SortedSetDocValues templateDocValues = DocValues.getSortedSet(leafReader, fieldType.templateFieldName()); + BinaryDocValues argsDocValues = getArgsDocValues(leafReader, fieldType.argsFieldName(), fieldType.useBinaryDocValuesArgs()); + SortedSetDocValues argsInfoDocValues = DocValues.getSortedSet(leafReader, fieldType.argsInfoFieldName()); + var docValues = new PatternTextDocValues(templateDocValues, argsDocValues, argsInfoDocValues); FieldInfo fieldInfo = leafReader.getFieldInfos().fieldInfo(fieldType.storedNamed()); if (fieldInfo == null) { + // If there is no stored subfield (either binary doc values or stored field), + // then there is no need to use PatternTextFallbackDocValues return docValues; } + // load binary doc values (for newer indices that store raw values in binary doc values) if (fieldInfo.getDocValuesType() == DocValuesType.BINARY) { BinaryDocValues rawBinaryDocValues = leafReader.getBinaryDocValues(fieldType.storedNamed()); return new BinaryFallback(docValues, templateIdDocValues, rawBinaryDocValues); } else { + // load stored field loader (for older indices that store raw values in stored fields) StoredFieldLoader storedFieldLoader = StoredFieldLoader.create(false, Set.of(fieldType.storedNamed())); LeafStoredFieldLoader storedTemplateLoader = storedFieldLoader.getLoader(leafReader.getContext(), null); return new LegacyStoredFieldFallback(storedTemplateLoader, fieldType.storedNamed(), docValues, templateIdDocValues); From d31fb5a192b07680cc829741269b1dc23c48994d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 24 Feb 2026 19:33:17 +0000 Subject: [PATCH 12/19] [CI] Auto commit changes from spotless --- .../blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java | 1 - .../xpack/logsdb/patterntext/PatternTextFieldMapperTests.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java index c88dd69db89bf..50f4f15c45c5f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper.blockloader.docvalues; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOFunction; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 2ff37b3c94f76..dfbcfc3e61e6f 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -20,10 +20,10 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; From 9019b19876c00c93bdc7e3d12964253bd30688c5 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 16:11:57 -0600 Subject: [PATCH 13/19] Fix Source-confirmed queries bug in separate PR --- .../patterntext/PatternTextFieldType.java | 2 +- .../PatternTextFieldMapperTests.java | 21 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index 9edce7b5577db..588dad4e0b9cf 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -153,7 +153,7 @@ private IOFunction, IOExcepti var docValues = PatternTextFallbackDocValues.from(context, PatternTextFieldType.this); return docId -> { if (docValues != null && docValues.advanceExact(docId)) { - return List.of(docValues.binaryValue().utf8ToString()); + return List.of(docValues.binaryValue()); } return List.of(); }; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index dfbcfc3e61e6f..2d7c90ef442c9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -136,27 +136,6 @@ private void assertPhraseQuery(MapperService mapperService) throws IOException { } } - public void testIntervalsQueryWithDisabledTemplating() throws IOException { - MapperService mapperService = createMapperService( - fieldMapping(b -> b.field("type", "pattern_text").field("disable_templating", true)) - ); - try (Directory directory = newDirectory()) { - RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "the quick brown fox 1"))).rootDoc(); - iw.addDocument(doc); - iw.close(); - try (DirectoryReader reader = DirectoryReader.open(directory)) { - SearchExecutionContext context = createSearchExecutionContext(mapperService, newSearcher(reader)); - PatternTextFieldType ft = (PatternTextFieldType) mapperService.fieldType("field"); - IntervalsSource intervalsSource = ft.termIntervals(new BytesRef("brown"), context); - Query query = new IntervalQuery("field", intervalsSource); - TopDocs docs = context.searcher().search(query, 1); - assertThat(docs.totalHits.value(), equalTo(1L)); - assertThat(docs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); - } - } - } - @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck( From 90af7396212de9a5bbff072452d2d6b6571691db Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 24 Feb 2026 22:19:48 +0000 Subject: [PATCH 14/19] [CI] Auto commit changes from spotless --- .../xpack/logsdb/patterntext/PatternTextFieldMapperTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 2d7c90ef442c9..9a5a34ffd7500 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -13,8 +13,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; -import org.apache.lucene.queries.intervals.IntervalQuery; -import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; @@ -23,7 +21,6 @@ import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; From 4198a0d2054c960a62067f3e4138cdcd5d61e01a Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 19:33:57 -0600 Subject: [PATCH 15/19] Revert "Fix Source-confirmed queries bug in separate PR" This reverts commit 9019b19876c00c93bdc7e3d12964253bd30688c5. --- .../patterntext/PatternTextFieldType.java | 2 +- .../PatternTextFieldMapperTests.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index 588dad4e0b9cf..9edce7b5577db 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -153,7 +153,7 @@ private IOFunction, IOExcepti var docValues = PatternTextFallbackDocValues.from(context, PatternTextFieldType.this); return docId -> { if (docValues != null && docValues.advanceExact(docId)) { - return List.of(docValues.binaryValue()); + return List.of(docValues.binaryValue().utf8ToString()); } return List.of(); }; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 9a5a34ffd7500..605c59f70a35d 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -133,6 +133,27 @@ private void assertPhraseQuery(MapperService mapperService) throws IOException { } } + public void testIntervalsQueryWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "pattern_text").field("disable_templating", true)) + ); + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "the quick brown fox 1"))).rootDoc(); + iw.addDocument(doc); + iw.close(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SearchExecutionContext context = createSearchExecutionContext(mapperService, newSearcher(reader)); + PatternTextFieldType ft = (PatternTextFieldType) mapperService.fieldType("field"); + IntervalsSource intervalsSource = ft.termIntervals(new BytesRef("brown"), context); + Query query = new IntervalQuery("field", intervalsSource); + TopDocs docs = context.searcher().search(query, 1); + assertThat(docs.totalHits.value(), equalTo(1L)); + assertThat(docs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); + } + } + } + @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck( From ffe707a7d8f23f8b3297ed39866c401c16b718bf Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Tue, 24 Feb 2026 19:55:50 -0600 Subject: [PATCH 16/19] missing import --- .../xpack/logsdb/patterntext/PatternTextFieldMapperTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 605c59f70a35d..dfbcfc3e61e6f 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -13,6 +13,8 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.queries.intervals.IntervalQuery; +import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; @@ -21,6 +23,7 @@ import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; From 2c25ef9f023aa461c43d73bcb1884fbba4a38198 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 25 Feb 2026 11:18:41 -0600 Subject: [PATCH 17/19] Add comment --- .../logsdb/patterntext/PatternTextFallbackDocValues.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java index 6c7c280413325..d8f2ef81f3b15 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java @@ -181,6 +181,12 @@ static BinaryDocValues fromEnabledPatternText(LeafReader leafReader, PatternText } } + /** + * It is not ideal that we are wrapping the stored fields loader in binary doc values. This is acceptable for this use case + * because combining a pattern_text doc values with its fallbacks is complicated and error-prone. Doing this in a single + * location reduces the chance of errors. Also, stored fields are only used as a fallback for proper pattern_text, either when + * a value exceeds 32kb, or templating is disabled. + */ private static BinaryDocValues storedFieldAsBinaryDocValues(LeafReaderContext context, String fieldName) throws IOException { var loader = StoredFieldLoader.create(false, Set.of(fieldName)); var leafLoader = loader.getLoader(context, null); From 63a9e9bbc0b0de7b7ec5bcaf637c33a9a78eb345 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 25 Feb 2026 11:20:31 -0600 Subject: [PATCH 18/19] Update docs/changelog/142767.yaml --- docs/changelog/142767.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/142767.yaml diff --git a/docs/changelog/142767.yaml b/docs/changelog/142767.yaml new file mode 100644 index 0000000000000..4ae7218e54034 --- /dev/null +++ b/docs/changelog/142767.yaml @@ -0,0 +1,5 @@ +area: Codec +issues: [] +pr: 142767 +summary: Fix NPE when querying `pattern_text` field in segment with no field values +type: bug From f3de3c6bd7b834afde99b112b5ead2f4a886dc61 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Wed, 25 Feb 2026 13:46:50 -0600 Subject: [PATCH 19/19] Fix flaky multi-segment PatternText tests Use NoMergePolicy with a direct IndexWriter instead of withLuceneIndex (which uses RandomIndexWriter that can randomly merge segments), ensuring tests that require separate segments per document are deterministic. --- .../PatternTextFieldMapperTests.java | 270 +++++++++--------- 1 file changed, 142 insertions(+), 128 deletions(-) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index dfbcfc3e61e6f..6865118ae9372 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -11,8 +11,11 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.queries.intervals.IntervalQuery; import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.FieldExistsQuery; @@ -499,58 +502,57 @@ public void testValueFetcherWithMissingFieldSegment() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); MappedFieldType ft = mapperService.fieldType("field"); - withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); - iw.addDocument(doc1); - iw.commit(); - - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); - iw.addDocument(doc2); - iw.commit(); - }, reader -> { - assertEquals(2, reader.leaves().size()); + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); - SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); - ValueFetcher fetcher = ft.valueFetcher(ctx, null); + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); - fetcher.setNextReader(reader.leaves().get(0)); - List values = fetcher.fetchValues(null, 0, new ArrayList<>()); - assertEquals(1, values.size()); - assertEquals("abc 123", values.get(0)); + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("abc 123", values.get(0)); - fetcher.setNextReader(reader.leaves().get(1)); - List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); - assertEquals(0, emptyValues.size()); - }); + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + } + } } public void testFieldDataWithMissingFieldSegment() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); MappedFieldType ft = mapperService.fieldType("field"); - withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); - iw.addDocument(doc1); - iw.commit(); - - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); - iw.addDocument(doc2); - iw.commit(); - }, reader -> { - assertEquals(2, reader.leaves().size()); + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); - var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); - var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); - var leafData0 = fieldData.load(reader.leaves().get(0)); - var bytesValues0 = leafData0.getBytesValues(); - assertTrue(bytesValues0.advanceExact(0)); - assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); - var leafData1 = fieldData.load(reader.leaves().get(1)); - var bytesValues1 = leafData1.getBytesValues(); - assertFalse(bytesValues1.advanceExact(0)); - }); + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + } + } } public void testValueFetcherWithDisabledTemplating() throws IOException { @@ -560,30 +562,29 @@ public void testValueFetcherWithDisabledTemplating() throws IOException { ); MappedFieldType ft = mapperService.fieldType("field"); - withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); - iw.addDocument(doc1); - iw.commit(); - - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc(); - iw.addDocument(doc2); - iw.commit(); - }, reader -> { - assertEquals(2, reader.leaves().size()); - - SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); - ValueFetcher fetcher = ft.valueFetcher(ctx, null); - - fetcher.setNextReader(reader.leaves().get(0)); - List values0 = fetcher.fetchValues(null, 0, new ArrayList<>()); - assertEquals(1, values0.size()); - assertEquals("abc 123", values0.get(0)); - - fetcher.setNextReader(reader.leaves().get(1)); - List values1 = fetcher.fetchValues(null, 0, new ArrayList<>()); - assertEquals(1, values1.size()); - assertEquals("foo 12", values1.get(0)); - }); + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values0 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values0.size()); + assertEquals("abc 123", values0.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List values1 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values1.size()); + assertEquals("foo 12", values1.get(0)); + } + } } public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throws IOException { @@ -593,29 +594,28 @@ public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throw ); MappedFieldType ft = mapperService.fieldType("field"); - withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); - iw.addDocument(doc1); - iw.commit(); - - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); - iw.addDocument(doc2); - iw.commit(); - }, reader -> { - assertEquals(2, reader.leaves().size()); + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); - SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); - ValueFetcher fetcher = ft.valueFetcher(ctx, null); + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); - fetcher.setNextReader(reader.leaves().get(0)); - List values = fetcher.fetchValues(null, 0, new ArrayList<>()); - assertEquals(1, values.size()); - assertEquals("abc 123", values.get(0)); + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("abc 123", values.get(0)); - fetcher.setNextReader(reader.leaves().get(1)); - List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); - assertEquals(0, emptyValues.size()); - }); + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + } + } } public void testFieldDataWithDisabledTemplating() throws IOException { @@ -625,29 +625,29 @@ public void testFieldDataWithDisabledTemplating() throws IOException { ); MappedFieldType ft = mapperService.fieldType("field"); - withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); - iw.addDocument(doc1); - iw.commit(); - - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc(); - iw.addDocument(doc2); - iw.commit(); - }, reader -> { - assertEquals(2, reader.leaves().size()); + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); - var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); - var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); - var leafData0 = fieldData.load(reader.leaves().get(0)); - var bytesValues0 = leafData0.getBytesValues(); - assertTrue(bytesValues0.advanceExact(0)); - assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); - var leafData1 = fieldData.load(reader.leaves().get(1)); - var bytesValues1 = leafData1.getBytesValues(); - assertFalse(bytesValues1.advanceExact(0)); - }); + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + } + } } public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOException { @@ -657,30 +657,44 @@ public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOExcep ); MappedFieldType ft = mapperService.fieldType("field"); - withLuceneIndex(mapperService, iw -> { - LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(); - iw.addDocument(doc1); - iw.commit(); - - LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc(); - iw.addDocument(doc2); - iw.commit(); - }, reader -> { - assertEquals(2, reader.leaves().size()); - - var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); - var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); - - var leafData0 = fieldData.load(reader.leaves().get(0)); - var bytesValues0 = leafData0.getBytesValues(); - assertTrue(bytesValues0.advanceExact(0)); - assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); - - var leafData1 = fieldData.load(reader.leaves().get(1)); - var bytesValues1 = leafData1.getBytesValues(); - assertTrue(bytesValues1.advanceExact(0)); - assertEquals("foo 12", bytesValues1.nextValue().utf8ToString()); - }); + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertTrue(bytesValues1.advanceExact(0)); + assertEquals("foo 12", bytesValues1.nextValue().utf8ToString()); + } + } + } + + /** + * Writes each document into its own segment with no merging, guaranteeing one leaf per doc. + */ + private static void indexDocPerSegment(Directory dir, LuceneDocument... docs) throws IOException { + IndexWriterConfig iwc = new IndexWriterConfig(); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter iw = new IndexWriter(dir, iwc)) { + for (LuceneDocument doc : docs) { + iw.addDocument(doc); + iw.commit(); + } + } } @Override