Skip to content

Fix NPE when querying pattern_text field in segment with no field values #142767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
parkertimmins merged 22 commits into from
Feb 25, 2026
Merged

Fix NPE when querying pattern_text field in segment with no field values #142767

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
7e43341
Add tests which NPE segment has no pattern_text values
parkertimmins Feb 20, 2026
23fddb3
Return empty doc values if segment has not pattern_text values
parkertimmins Feb 20, 2026
068997c
Add tests for pattern_text with disabled templating
parkertimmins Feb 20, 2026
013aae6
Fix pattern_text value loading with disabled templating
parkertimmins Feb 20, 2026
e1cbc1b
Use numbers in test values so will produce pattern_text arg tokens
parkertimmins Feb 20, 2026
3a0d24b
add removed comment
parkertimmins Feb 20, 2026
6f565d8
Fix pattern_text source-confirmed queries returning BytesRef
parkertimmins Feb 20, 2026
4e34637
Merge branch 'main' into parker/pattern-text-empty-segment-npe
parkertimmins Feb 20, 2026
8a70bdf
Add intervals query test for disabled templating
parkertimmins Feb 24, 2026
9aace57
move from method to PatternTextDocValues
parkertimmins Feb 24, 2026
1298c2e
Centralize doc values loading in PatternTextFallbackDocValues
parkertimmins Feb 24, 2026
90c5199
add back some comments
parkertimmins Feb 24, 2026
d31fb5a
[CI] Auto commit changes from spotless
Feb 24, 2026
9019b19
Fix Source-confirmed queries bug in separate PR
parkertimmins Feb 24, 2026
90af739
[CI] Auto commit changes from spotless
Feb 24, 2026
32471d9
Merge branch 'main' into parker/pattern-text-empty-segment-npe
parkertimmins Feb 24, 2026
4198a0d
Revert "Fix Source-confirmed queries bug in separate PR"
parkertimmins Feb 25, 2026
ffe707a
missing import
parkertimmins Feb 25, 2026
2c25ef9
Add comment
parkertimmins Feb 25, 2026
63a9e9b
Update docs/changelog/142767.yaml
parkertimmins Feb 25, 2026
68f599b
Merge branch 'main' into parker/pattern-text-empty-segment-npe
parkertimmins Feb 25, 2026
f3de3c6
Fix flaky multi-segment PatternText tests
parkertimmins Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 64 additions & 38 deletions ...logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -130,70 +129,93 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
@Override
public void setNextReader(LeafReaderContext context) {
try {
this.docValues = PatternTextFallbackDocValues.from(context.reader(), PatternTextFieldType.this);
this.docValues = loadDocValues(context);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

@Override
public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValues) throws IOException {
if (false == docValues.advanceExact(doc)) {
if (docValues == null || false == docValues.advanceExact(doc)) {
return List.of();
}
return List.of(docValues.binaryValue().utf8ToString());
}

@Override
public StoredFieldsSpec storedFieldsSpec() {
// PatternedTextCompositeValues may require a stored field, but it handles loading this field internally.
// Pattern Text may require a stored field, but it handles loading this field internally.
return StoredFieldsSpec.NO_REQUIREMENTS;
}
};
}

private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> getValueFetcherProvider(
SearchExecutionContext searchExecutionContext
) {
if (disableTemplating) {
return useBinaryDocValuesRawText ? binaryDocValuesFetcher(storedNamed()) : storedFieldFetcher(storedNamed());
}

return context -> {
ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null);
valueFetcher.setNextReader(context);
return docID -> {
try {
return valueFetcher.fetchValues(null, docID, new ArrayList<>());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
};
}

private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> binaryDocValuesFetcher(String name) {
private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> getValueFetcherProvider() {
return context -> {
var docValues = context.reader().getBinaryDocValues(name);
var docValues = loadDocValues(context);
return docId -> {
if (docValues != null && docValues.advanceExact(docId)) {
return List.of(docValues.binaryValue());
return List.of(docValues.binaryValue().utf8ToString());
}
return List.of();
};
};
}

private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> storedFieldFetcher(String name) {
var loader = StoredFieldLoader.create(false, Set.of(name));
return context -> {
var leafLoader = loader.getLoader(context, null);
return docId -> {
leafLoader.advanceTo(docId);
BinaryDocValues loadDocValues(LeafReaderContext context) throws IOException {
if (disableTemplating) {
if (useBinaryDocValuesRawText) {
return context.reader().getBinaryDocValues(storedNamed());
}
return storedFieldAsBinaryDocValues(context, storedNamed());
}
return PatternTextFallbackDocValues.from(context.reader(), this);
}

private static BinaryDocValues storedFieldAsBinaryDocValues(LeafReaderContext context, String fieldName) throws IOException {

@Kubik42 Kubik42 Feb 20, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This feels kind of wrong to do - converting stored fields into doc values. Are you doing this to return BinaryDocValues from loadDocValues()? Can we use PatternTextFallbackDocValues instead?

@parkertimmins parkertimmins Feb 20, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wrapping in doc values is just to simplify the valueFetcher logic. So PatternTextFallbackDocValues wraps the proper pattern_text, the binary fallback, and the stored fallback in a single binary doc values. loadDocValues does the same thing, but it makes the decision between the three options at the whole column level rather than on a per-doc basis. So it will have fewer branches since it doesn't require checking the main pattern_text iterator before falling back on each doc.

I think we'll want to wrap the stored field in a doc value iterator, but we might be able to push this down into PatternTextFallbackDocValues in a cleaner way. I'll give it some more thought next week.

@martijnvg martijnvg Feb 23, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If possible I think we should make this decision at the column level. And I think this is possible.

var loader = StoredFieldLoader.create(false, Set.of(fieldName));
var leafLoader = loader.getLoader(context, null);
return new BinaryDocValues() {
BytesRef currentValue;

@Override
public boolean advanceExact(int target) throws IOException {
leafLoader.advanceTo(target);
var storedFields = leafLoader.storedFields();
var values = storedFields.get(name);
return values != null ? values : List.of();
};
var fieldValues = storedFields.get(fieldName);
if (fieldValues != null && fieldValues.isEmpty() == false) {
currentValue = (BytesRef) fieldValues.getFirst();
return true;
}
currentValue = null;
return false;
}

@Override
public BytesRef binaryValue() {
return currentValue;
}

@Override
public int docID() {
throw new UnsupportedOperationException();
}

@Override
public int nextDoc() {
throw new UnsupportedOperationException();
}

@Override
public int advance(int target) {
throw new UnsupportedOperationException();
}

@Override
public long cost() {
return 0;
}
};
}

Expand All @@ -202,12 +224,12 @@ private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext contex
if (hasPositions) {
return new ConstantScoreQuery(query);
} else {
return new ConstantScoreQuery(new SourceConfirmedTextQuery(query, getValueFetcherProvider(context), indexAnalyzer));
return new ConstantScoreQuery(new SourceConfirmedTextQuery(query, getValueFetcherProvider(), indexAnalyzer));
}
}

private IntervalsSource toIntervalsSource(IntervalsSource source, Query approximation, SearchExecutionContext searchExecutionContext) {
return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(searchExecutionContext), indexAnalyzer);
return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(), indexAnalyzer);
}

@Override
Expand Down Expand Up @@ -380,4 +402,8 @@ boolean useBinaryDocValuesArgs() {
return useBinaryDocValuesArgs;
}

boolean useBinaryDocValuesRawText() {
return useBinaryDocValuesRawText;
}

}
7 changes: 3 additions & 4 deletions ...b/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

package org.elasticsearch.xpack.logsdb.patterntext;

import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -71,8 +71,7 @@ public LeafFieldData load(LeafReaderContext context) {

@Override
public LeafFieldData loadDirect(LeafReaderContext context) throws IOException {
LeafReader leafReader = context.reader();
var values = PatternTextFallbackDocValues.from(leafReader, fieldType);
final BinaryDocValues values = fieldType.loadDocValues(context);
return new LeafFieldData() {

final ToScriptFieldFactory<SortedBinaryDocValues> factory = KeywordDocValuesField::new;
Expand All @@ -87,7 +86,7 @@ public SortedBinaryDocValues getBytesValues() {
return new SortedBinaryDocValues() {
@Override
public boolean advanceExact(int doc) throws IOException {
return values.advanceExact(doc);
return values != null && values.advanceExact(doc);
}

@Override
Expand Down
188 changes: 188 additions & 0 deletions ...src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,194 @@ protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
}

public void testValueFetcherWithMissingFieldSegment() throws IOException {
MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text")));
MappedFieldType ft = mapperService.fieldType("field");

withLuceneIndex(mapperService, iw -> {
LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc();
iw.addDocument(doc1);
iw.commit();

LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc();
iw.addDocument(doc2);
iw.commit();
}, reader -> {
assertEquals(2, reader.leaves().size());

SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader));
ValueFetcher fetcher = ft.valueFetcher(ctx, null);

fetcher.setNextReader(reader.leaves().get(0));
List<Object> values = fetcher.fetchValues(null, 0, new ArrayList<>());
assertEquals(1, values.size());
assertEquals("abc 123", values.get(0));

fetcher.setNextReader(reader.leaves().get(1));
List<Object> emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>());
assertEquals(0, emptyValues.size());
});
}

public void testFieldDataWithMissingFieldSegment() throws IOException {
MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text")));
MappedFieldType ft = mapperService.fieldType("field");

withLuceneIndex(mapperService, iw -> {
LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc();
iw.addDocument(doc1);
iw.commit();

LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc();
iw.addDocument(doc2);
iw.commit();
}, reader -> {
assertEquals(2, reader.leaves().size());

var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT);
var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService());

var leafData0 = fieldData.load(reader.leaves().get(0));
var bytesValues0 = leafData0.getBytesValues();
assertTrue(bytesValues0.advanceExact(0));
assertEquals("abc 123", bytesValues0.nextValue().utf8ToString());

var leafData1 = fieldData.load(reader.leaves().get(1));
var bytesValues1 = leafData1.getBytesValues();
assertFalse(bytesValues1.advanceExact(0));
});
}

public void testValueFetcherWithDisabledTemplating() throws IOException {
MapperService mapperService = createMapperService(
Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(),
fieldMapping(b -> b.field("type", "pattern_text"))
);
MappedFieldType ft = mapperService.fieldType("field");

withLuceneIndex(mapperService, iw -> {
LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc();
iw.addDocument(doc1);
iw.commit();

LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc();
iw.addDocument(doc2);
iw.commit();
}, reader -> {
assertEquals(2, reader.leaves().size());

SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader));
ValueFetcher fetcher = ft.valueFetcher(ctx, null);

fetcher.setNextReader(reader.leaves().get(0));
List<Object> values0 = fetcher.fetchValues(null, 0, new ArrayList<>());
assertEquals(1, values0.size());
assertEquals("abc 123", values0.get(0));

fetcher.setNextReader(reader.leaves().get(1));
List<Object> values1 = fetcher.fetchValues(null, 0, new ArrayList<>());
assertEquals(1, values1.size());
assertEquals("foo 12", values1.get(0));
});
}

public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throws IOException {
MapperService mapperService = createMapperService(
Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(),
fieldMapping(b -> b.field("type", "pattern_text"))
);
MappedFieldType ft = mapperService.fieldType("field");

withLuceneIndex(mapperService, iw -> {
LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc();
iw.addDocument(doc1);
iw.commit();

LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc();
iw.addDocument(doc2);
iw.commit();
}, reader -> {
assertEquals(2, reader.leaves().size());

SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader));
ValueFetcher fetcher = ft.valueFetcher(ctx, null);

fetcher.setNextReader(reader.leaves().get(0));
List<Object> values = fetcher.fetchValues(null, 0, new ArrayList<>());
assertEquals(1, values.size());
assertEquals("abc 123", values.get(0));

fetcher.setNextReader(reader.leaves().get(1));
List<Object> emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>());
assertEquals(0, emptyValues.size());
});
}

public void testFieldDataWithDisabledTemplating() throws IOException {
MapperService mapperService = createMapperService(
Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(),
fieldMapping(b -> b.field("type", "pattern_text"))
);
MappedFieldType ft = mapperService.fieldType("field");

withLuceneIndex(mapperService, iw -> {
LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc();
iw.addDocument(doc1);
iw.commit();

LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> {})).rootDoc();
iw.addDocument(doc2);
iw.commit();
}, reader -> {
assertEquals(2, reader.leaves().size());

var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT);
var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService());

var leafData0 = fieldData.load(reader.leaves().get(0));
var bytesValues0 = leafData0.getBytesValues();
assertTrue(bytesValues0.advanceExact(0));
assertEquals("abc 123", bytesValues0.nextValue().utf8ToString());

var leafData1 = fieldData.load(reader.leaves().get(1));
var bytesValues1 = leafData1.getBytesValues();
assertFalse(bytesValues1.advanceExact(0));
});
}

public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOException {
MapperService mapperService = createMapperService(
Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(),
fieldMapping(b -> b.field("type", "pattern_text"))
);
MappedFieldType ft = mapperService.fieldType("field");

withLuceneIndex(mapperService, iw -> {
LuceneDocument doc1 = mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc();
iw.addDocument(doc1);
iw.commit();

LuceneDocument doc2 = mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc();
iw.addDocument(doc2);
iw.commit();
}, reader -> {
assertEquals(2, reader.leaves().size());

var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT);
var fieldData = ft.fielddataBuilder(fieldDataContext).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService());

var leafData0 = fieldData.load(reader.leaves().get(0));
var bytesValues0 = leafData0.getBytesValues();
assertTrue(bytesValues0.advanceExact(0));
assertEquals("abc 123", bytesValues0.nextValue().utf8ToString());

var leafData1 = fieldData.load(reader.leaves().get(1));
var bytesValues1 = leafData1.getBytesValues();
assertTrue(bytesValues1.advanceExact(0));
assertEquals("foo 12", bytesValues1.nextValue().utf8ToString());
});
}

@Override
protected List<SortShortcutSupport> getSortShortcutSupport() {
return List.of();
Expand Down