diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java index bb1a9efc2fc6a..0076c1cc9f23d 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldType.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockStoredFieldsReader; @@ -119,6 +120,9 @@ public String familyTypeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (disableTemplating) { + return storedFieldValueFetcher(); + } return new ValueFetcher() { BinaryDocValues docValues; @@ -133,7 +137,7 @@ public void setNextReader(LeafReaderContext context) { @Override public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { - if (false == docValues.advanceExact(doc)) { + if (docValues == null || false == docValues.advanceExact(doc)) { return List.of(); } return List.of(docValues.binaryValue().utf8ToString()); @@ -141,7 +145,39 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - // PatternedTextCompositeValues may require a stored field, but it handles loading this field internally. + // PatternTextCompositeValues may require a stored field, but it handles loading this field internally. + return StoredFieldsSpec.NO_REQUIREMENTS; + } + }; + } + + private ValueFetcher storedFieldValueFetcher() { + var loader = StoredFieldLoader.create(false, Set.of(storedNamed())); + return new ValueFetcher() { + LeafStoredFieldLoader leafLoader; + + @Override + public void setNextReader(LeafReaderContext context) { + try { + this.leafLoader = loader.getLoader(context, null); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { + leafLoader.advanceTo(doc); + var storedFields = leafLoader.storedFields(); + var values = storedFields.get(storedNamed()); + if (values == null || values.isEmpty()) { + return List.of(); + } + return List.of(((BytesRef) values.getFirst()).utf8ToString()); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } }; @@ -174,7 +210,11 @@ private static IOFunction, IO return docId -> { leafLoader.advanceTo(docId); var storedFields = leafLoader.storedFields(); - return storedFields.get(name); + var values = storedFields.get(name); + if (values == null || values.isEmpty()) { + return List.of(); + } + return List.of(((BytesRef) values.getFirst()).utf8ToString()); }; }; } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java index 2d09f1f44e21f..99f4e69c3f1bb 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextIndexFieldData.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.logsdb.patterntext; -import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; @@ -16,6 +16,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.KeywordDocValuesField; @@ -28,6 +29,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Set; public class PatternTextIndexFieldData implements IndexFieldData { @@ -71,8 +73,12 @@ public LeafFieldData load(LeafReaderContext context) { @Override public LeafFieldData loadDirect(LeafReaderContext context) throws IOException { - LeafReader leafReader = context.reader(); - var values = PatternTextCompositeValues.from(leafReader, fieldType); + final BinaryDocValues values; + if (fieldType.disableTemplating()) { + values = loadStoredFieldDocValues(context); + } else { + values = PatternTextCompositeValues.from(context.reader(), fieldType); + } return new LeafFieldData() { final ToScriptFieldFactory factory = KeywordDocValuesField::new; @@ -87,7 +93,7 @@ public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { @Override public boolean advanceExact(int doc) throws IOException { - return values.advanceExact(doc); + return values != null && values.advanceExact(doc); } @Override @@ -109,6 +115,52 @@ public long ramBytesUsed() { }; } + private BinaryDocValues loadStoredFieldDocValues(LeafReaderContext context) throws IOException { + var loader = StoredFieldLoader.create(false, Set.of(fieldType.storedNamed())); + var leafLoader = loader.getLoader(context, null); + return new BinaryDocValues() { + BytesRef currentValue; + + @Override + public boolean advanceExact(int target) throws IOException { + leafLoader.advanceTo(target); + var storedFields = leafLoader.storedFields(); + var fieldValues = storedFields.get(fieldType.storedNamed()); + if (fieldValues != null && fieldValues.isEmpty() == false) { + currentValue = (BytesRef) fieldValues.getFirst(); + return true; + } + currentValue = null; + return false; + } + + @Override + public BytesRef binaryValue() { + return currentValue; + } + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return 0; + } + }; + } + @Override public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { throw new IllegalArgumentException("not supported for source pattern_text field type"); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 53895e9cea8b7..d5b8d6652f276 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -11,8 +11,13 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.queries.intervals.IntervalQuery; +import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; @@ -21,6 +26,7 @@ import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -466,6 +472,226 @@ public void testSyntheticSourceKeepArrays() { // This mapper does not allow arrays } + public void testIntervalsQueryWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "pattern_text").field("disable_templating", true)) + ); + try (Directory directory = newDirectory()) { + RandomIndexWriter iw = new RandomIndexWriter(random(), directory); + LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "the quick brown fox 1"))).rootDoc(); + iw.addDocument(doc); + iw.close(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + SearchExecutionContext context = createSearchExecutionContext(mapperService, newSearcher(reader)); + PatternTextFieldType ft = (PatternTextFieldType) mapperService.fieldType("field"); + IntervalsSource intervalsSource = ft.termIntervals(new BytesRef("brown"), context); + Query query = new IntervalQuery("field", intervalsSource); + TopDocs docs = context.searcher().search(query, 1); + assertThat(docs.totalHits.value(), equalTo(1L)); + assertThat(docs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); + } + } + } + + public void testValueFetcherWithMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("abc 123", values.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + } + } + } + + public void testFieldDataWithMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "pattern_text"))); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + } + } + } + + public void testValueFetcherWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values0 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values0.size()); + assertEquals("abc 123", values0.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List values1 = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values1.size()); + assertEquals("foo 12", values1.get(0)); + } + } + } + + public void testValueFetcherWithDisabledTemplatingAndMissingFieldSegment() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + SearchExecutionContext ctx = createSearchExecutionContext(mapperService, newSearcher(reader)); + ValueFetcher fetcher = ft.valueFetcher(ctx, null); + + fetcher.setNextReader(reader.leaves().get(0)); + List values = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(1, values.size()); + assertEquals("abc 123", values.get(0)); + + fetcher.setNextReader(reader.leaves().get(1)); + List emptyValues = fetcher.fetchValues(null, 0, new ArrayList<>()); + assertEquals(0, emptyValues.size()); + } + } + } + + public void testFieldDataWithDisabledTemplating() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> {})).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertFalse(bytesValues1.advanceExact(0)); + } + } + } + + public void testFieldDataWithDisabledTemplatingAllDocsHaveField() throws IOException { + MapperService mapperService = createMapperService( + Settings.builder().put("index.mapping.pattern_text.disable_templating", true).build(), + fieldMapping(b -> b.field("type", "pattern_text")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + + try (Directory dir = newDirectory()) { + indexDocPerSegment( + dir, + mapperService.documentMapper().parse(source(b -> b.field("field", "abc 123"))).rootDoc(), + mapperService.documentMapper().parse(source(b -> b.field("field", "foo 12"))).rootDoc() + ); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + + var fieldDataContext = new FieldDataContext("", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + var fieldData = ft.fielddataBuilder(fieldDataContext) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + + var leafData0 = fieldData.load(reader.leaves().get(0)); + var bytesValues0 = leafData0.getBytesValues(); + assertTrue(bytesValues0.advanceExact(0)); + assertEquals("abc 123", bytesValues0.nextValue().utf8ToString()); + + var leafData1 = fieldData.load(reader.leaves().get(1)); + var bytesValues1 = leafData1.getBytesValues(); + assertTrue(bytesValues1.advanceExact(0)); + assertEquals("foo 12", bytesValues1.nextValue().utf8ToString()); + } + } + } + + /** + * Writes each document into its own segment with no merging, guaranteeing one leaf per doc. + */ + private static void indexDocPerSegment(Directory dir, LuceneDocument... docs) throws IOException { + IndexWriterConfig iwc = new IndexWriterConfig(); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter iw = new IndexWriter(dir, iwc)) { + for (LuceneDocument doc : docs) { + iw.addDocument(doc); + iw.commit(); + } + } + } + @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported");