diff --git a/docs/changelog/142767.yaml b/docs/changelog/142767.yaml new file mode 100644 index 0000000000000..4ae7218e54034 --- /dev/null +++ b/docs/changelog/142767.yaml @@ -0,0 +1,5 @@ +area: Codec +issues: [] +pr: 142767 +summary: Fix NPE when querying `pattern_text` field in segment with no field values +type: bug diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java index 59de237bebbb9..b33441d1643cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper.blockloader.docvalues; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOFunction; @@ -27,10 +26,10 @@ * {@link BytesRefsFromOrdsBlockLoader} for ordinals-based binary values */ public class BytesRefsFromBinaryBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { - private final IOFunction docValuesSupplier; + private final IOFunction docValuesSupplier; public BytesRefsFromBinaryBlockLoader(String fieldName) { - this(leafReader -> leafReader.getBinaryDocValues(fieldName)); + this(context -> context.reader().getBinaryDocValues(fieldName)); } /** @@ -38,7 +37,7 @@ public BytesRefsFromBinaryBlockLoader(String fieldName) { * This is useful when the doc values are not directly stored in a single field * but are composed of multiple sources, as is the case for Pattern Text. */ - public BytesRefsFromBinaryBlockLoader(IOFunction docValuesSupplier) { + public BytesRefsFromBinaryBlockLoader(IOFunction docValuesSupplier) { this.docValuesSupplier = docValuesSupplier; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/tracking/TrackingBinaryDocValues.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/tracking/TrackingBinaryDocValues.java index 4c805d7f81395..40b9db5442e64 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/tracking/TrackingBinaryDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/tracking/TrackingBinaryDocValues.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper.blockloader.docvalues.tracking; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.IOFunction; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -31,18 +30,18 @@ public record TrackingBinaryDocValues(CircuitBreaker breaker, BinaryDocValues do private static final long ESTIMATED_SIZE = ByteSizeValue.ofKb(3).getBytes(); public static TrackingBinaryDocValues get(CircuitBreaker breaker, LeafReaderContext context, String fieldName) throws IOException { - return get(breaker, context, leafReader -> leafReader.getBinaryDocValues(fieldName)); + return get(breaker, context, ctx -> ctx.reader().getBinaryDocValues(fieldName)); } public static TrackingBinaryDocValues get( CircuitBreaker breaker, LeafReaderContext context, - IOFunction supplier + IOFunction supplier ) throws IOException { breaker.addEstimateBytesAndMaybeBreak(ESTIMATED_SIZE, "load blocks"); TrackingBinaryDocValues result = null; try { - BinaryDocValues docValues = supplier.apply(context.reader()); + BinaryDocValues docValues = supplier.apply(context); if (docValues == null) { return null; } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java index 1456c44615773..d8f2ef81f3b15 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFallbackDocValues.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; @@ -131,7 +132,26 @@ public boolean advanceExact(int target) throws IOException { } } - static BinaryDocValues from(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException { + /** + * Returns doc values for a pattern_text field. If templating is disabled returns an alternative version consisting of either + * a flat binary doc values column or stored fields. The decision between proper pattern_text, flat binary doc values, or + * stored fields occurs at the column level. Returns null if the segment has no values for this field. + */ + static BinaryDocValues from(LeafReaderContext context, PatternTextFieldType fieldType) throws IOException { + if (fieldType.disableTemplating()) { + if (fieldType.useBinaryDocValuesRawText()) { + return context.reader().getBinaryDocValues(fieldType.storedNamed()); + } + return storedFieldAsBinaryDocValues(context, fieldType.storedNamed()); + } + return fromEnabledPatternText(context.reader(), fieldType); + } + + /** + * Returns doc values for a pattern_text field that has templating enabled. Single rows may fall back to binary doc values or a + * stored field. This decision happens on a per-doc basis. Returns null if the segment has no template_id values. + */ + static BinaryDocValues fromEnabledPatternText(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException { SortedSetDocValues templateIdDocValues = DocValues.getSortedSet(leafReader, fieldType.templateIdFieldName()); if (templateIdDocValues.getValueCount() == 0) { return null; @@ -160,4 +180,56 @@ static BinaryDocValues from(LeafReader leafReader, PatternTextFieldType fieldTyp return new LegacyStoredFieldFallback(storedTemplateLoader, fieldType.storedNamed(), docValues, templateIdDocValues); } } + + /** + * It is not ideal that we are wrapping the stored fields loader in binary doc values. This is acceptable for this use case + * because combining a pattern_text doc values with its fallbacks is complicated and error-prone. Doing this in a single + * location reduces the chance of errors. Also, stored fields are only used as a fallback for proper pattern_text, either when + * a value exceeds 32kb, or templating is disabled. + */ + private static BinaryDocValues storedFieldAsBinaryDocValues(LeafReaderContext context, String fieldName) throws IOException { + var loader = StoredFieldLoader.create(false, Set.of(fieldName)); + var leafLoader = loader.getLoader(context, null); + return new BinaryDocValues() { + BytesRef currentValue; + + @Override + public boolean advanceExact(int target) throws IOException { + leafLoader.advanceTo(target); + var storedFields = leafLoader.storedFields(); + var fieldValues = storedFields.get(fieldName); + if (fieldValues != null && fieldValues.isEmpty() == false) { + currentValue = (BytesRef) fieldValues.getFirst(); + return true; + } + currentValue = null; + return false; + } + + @Override + public BytesRef binaryValue() { + return currentValue; + } + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return 0; + } + }; + } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java index 0b49cb6f40b54..0d7f1ab4b4a2b 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java @@ -391,7 +391,7 @@ protected void write(XContentBuilder b, Object value) throws IOException { fullPath(), new PatternTextSyntheticFieldLoaderLayer( fieldType().name(), - leafReader -> PatternTextFallbackDocValues.from(leafReader, fieldType()) + leafReader -> PatternTextFallbackDocValues.fromEnabledPatternText(leafReader, fieldType()) ) ); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index 24ce0055e2dc8..9edce7b5577db 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -29,9 +29,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.BlockStoredFieldsReader; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -46,12 +44,10 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; public class PatternTextFieldType extends TextFamilyFieldType { @@ -130,7 +126,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public void setNextReader(LeafReaderContext context) { try { - this.docValues = PatternTextFallbackDocValues.from(context.reader(), PatternTextFieldType.this); + this.docValues = PatternTextFallbackDocValues.from(context, PatternTextFieldType.this); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -138,7 +134,7 @@ public void setNextReader(LeafReaderContext context) { @Override public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { - if (false == docValues.advanceExact(doc)) { + if (docValues == null || false == docValues.advanceExact(doc)) { return List.of(); } return List.of(docValues.binaryValue().utf8ToString()); @@ -146,68 +142,35 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - // PatternedTextCompositeValues may require a stored field, but it handles loading this field internally. + // Pattern Text may require a stored field, but it handles loading this field internally. return StoredFieldsSpec.NO_REQUIREMENTS; } }; } - private IOFunction, IOException>> getValueFetcherProvider( - SearchExecutionContext searchExecutionContext - ) { - if (disableTemplating) { - return useBinaryDocValuesRawText ? binaryDocValuesFetcher(storedNamed()) : storedFieldFetcher(storedNamed()); - } - - return context -> { - ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null); - valueFetcher.setNextReader(context); - return docID -> { - try { - return valueFetcher.fetchValues(null, docID, new ArrayList<>()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }; - }; - } - - private static IOFunction, IOException>> binaryDocValuesFetcher(String name) { + private IOFunction, IOException>> getValueFetcherProvider() { return context -> { - var docValues = context.reader().getBinaryDocValues(name); + var docValues = PatternTextFallbackDocValues.from(context, PatternTextFieldType.this); return docId -> { if (docValues != null && docValues.advanceExact(docId)) { - return List.of(docValues.binaryValue()); + return List.of(docValues.binaryValue().utf8ToString()); } return List.of(); }; }; } - private static IOFunction, IOException>> storedFieldFetcher(String name) { - var loader = StoredFieldLoader.create(false, Set.of(name)); - return context -> { - var leafLoader = loader.getLoader(context, null); - return docId -> { - leafLoader.advanceTo(docId); - var storedFields = leafLoader.storedFields(); - var values = storedFields.get(name); - return values != null ? values : List.of(); - }; - }; - } - private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext context) { // Disable scoring similarly to match_only_text if (hasPositions) { return new ConstantScoreQuery(query); } else { - return new ConstantScoreQuery(new SourceConfirmedTextQuery(query, getValueFetcherProvider(context), indexAnalyzer)); + return new ConstantScoreQuery(new SourceConfirmedTextQuery(query, getValueFetcherProvider(), indexAnalyzer)); } } private IntervalsSource toIntervalsSource(IntervalsSource source, Query approximation, SearchExecutionContext searchExecutionContext) { - return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(searchExecutionContext), indexAnalyzer); + return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(), indexAnalyzer); } @Override @@ -327,16 +290,6 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (disableTemplating) { - if (useBinaryDocValuesRawText) { - // for newer indices, raw pattern text values are stored in binary doc values - return new BytesRefsFromBinaryBlockLoader(storedNamed()); - } else { - // for older indices (bwc), raw pattern text values are stored in stored fields - return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedNamed()); - } - } - return new BytesRefsFromBinaryBlockLoader(leafReader -> PatternTextFallbackDocValues.from(leafReader, this)); } @@ -380,4 +333,8 @@ boolean useBinaryDocValuesArgs() { return useBinaryDocValuesArgs; } + boolean useBinaryDocValuesRawText() { + return useBinaryDocValuesRawText; + } + } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java index c9b8cb0c5d1c7..516c2abdb3942 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.logsdb.patterntext; -import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; @@ -71,8 +71,7 @@ public LeafFieldData load(LeafReaderContext context) { @Override public LeafFieldData loadDirect(LeafReaderContext context) throws IOException { - LeafReader leafReader = context.reader(); - var values = PatternTextFallbackDocValues.from(leafReader, fieldType); + final BinaryDocValues values = PatternTextFallbackDocValues.from(context, fieldType); return new LeafFieldData() { final ToScriptFieldFactory factory = KeywordDocValuesField::new; @@ -87,7 +86,7 @@ public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { @Override public boolean advanceExact(int doc) throws IOException { - return values.advanceExact(doc); + return values != null && values.advanceExact(doc); } @Override diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 53895e9cea8b7..6865118ae9372 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -11,8 +11,13 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.queries.intervals.IntervalQuery; +import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; @@ -21,6 +26,7 @@ import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -133,6 +139,27 @@ private void assertPhraseQuery(MapperService mapperService) throws IOException { } } + public void testIntervalsQueryWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "pattern_text").field("disable_templating", true)) + ); + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "the quick brown fox 1"))).rootDoc(); + iw.addDocument(doc); + iw.close(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SearchExecutionContext context = createSearchExecutionContext(mapperService, newSearcher(reader)); + PatternTextFieldType ft = (PatternTextFieldType) mapperService.fieldType("field"); + IntervalsSource intervalsSource = ft.termIntervals(new BytesRef("brown"), context); + Query query = new IntervalQuery("field", intervalsSource); + TopDocs docs = context.searcher().search(query, 1); + assertThat(docs.totalHits.value(), equalTo(1L)); + assertThat(docs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); + } + } + } + @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck( @@ -471,6 +498,205 @@ protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + public void testValueFetcherWithMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("abc 123", values.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + } + } + } + + public void testFieldDataWithMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + } + } + } + + public void testValueFetcherWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values0 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values0.size()); + assertEquals("abc 123", values0.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List values1 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values1.size()); + assertEquals("foo 12", values1.get(0)); + } + } + } + + public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("abc 123", values.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + } + } + } + + public void testFieldDataWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + } + } + } + + public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertTrue(bytesValues1.advanceExact(0)); + assertEquals("foo 12", bytesValues1.nextValue().utf8ToString()); + } + } + } + + /** + * Writes each document into its own segment with no merging, guaranteeing one leaf per doc. + */ + private static void indexDocPerSegment(Directory dir, LuceneDocument... docs) throws IOException { + IndexWriterConfig iwc = new IndexWriterConfig(); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter iw = new IndexWriter(dir, iwc)) { + for (LuceneDocument doc : docs) { + iw.addDocument(doc); + iw.commit(); + } + } + } + @Override protected List getSortShortcutSupport() { return List.of(); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java index d74a61e401f4c..57391b13d44f5 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldTypeTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.BlockStoredFieldsReader; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromBinaryBlockLoader; @@ -266,7 +265,7 @@ public void testBlockLoaderWhenTemplatingIsDisabledAndStoredFieldsAreUsed() { false ); BlockLoader blockLoader = ft.blockLoader(null); - assertThat(blockLoader, Matchers.instanceOf(BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader.class)); + assertThat(blockLoader, Matchers.instanceOf(BytesRefsFromBinaryBlockLoader.class)); } }