From 82716e4b86d07c4781e38c628c4508d8fb34ceff Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 15 Sep 2025 09:58:30 -0700 Subject: [PATCH 1/3] Conditionally disable templating in patterned_text --- .../PatternedTextFieldMapper.java | 41 ++++++++++++++----- .../patternedtext/PatternedTextFieldType.java | 5 +++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java index a6cf55a29622f..c3d9ae30c314b 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java @@ -30,8 +30,11 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; import org.elasticsearch.index.mapper.TextParams; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; @@ -261,12 +264,17 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio throw new IllegalArgumentException("Multiple values are not allowed for field [" + fieldType().name() + "]."); } - // Parse template and args - PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value); - // Add index on original value context.doc().add(new Field(fieldType().name(), value, fieldType)); + if (fieldType().disableTemplating()) { + context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value))); + return; + } + + // Parse template and args + PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value); + // Add template_id doc_values context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId()))); @@ -305,14 +313,25 @@ interface DocValuesSupplier { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - return new SyntheticSourceSupport.Native( - () -> new CompositeSyntheticFieldLoader( - leafName(), - fullPath(), - new PatternedTextSyntheticFieldLoaderLayer( - fieldType().name(), - leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType()) - ) + return new SyntheticSourceSupport.Native(this::getSyntheticFieldLoader); + } + + private SourceLoader.SyntheticFieldLoader getSyntheticFieldLoader() { + if (fieldType().disableTemplating()) { + return new StringStoredFieldFieldLoader(fieldType().storedNamed(), fieldType().name(), leafName()) { + @Override + protected void write(XContentBuilder b, Object value) throws IOException { + b.value(((BytesRef) value).utf8ToString()); + } + }; + } + + return new CompositeSyntheticFieldLoader( + leafName(), + fullPath(), + new PatternedTextSyntheticFieldLoaderLayer( + fieldType().name(), + leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType()) ) ); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index 35f1047559e89..a4451b9cb58e4 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.BlockStoredFieldsReader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -262,6 +263,10 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { + if (disableTemplating) { + return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedNamed()); + } + return new PatternedTextBlockLoader((leafReader -> PatternedTextCompositeValues.from(leafReader, this))); } From daec41cfb41823c14d018d75ba2a63d02c6b4dd9 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 15 Sep 2025 11:39:07 -0700 Subject: [PATCH 2/3] Add patterned_text tests for disable_templating:true --- .../PatternedTextBasicRestIT.java | 18 ++++++- .../patternedtext/PatternedTextFieldType.java | 18 +++++++ .../PatternedTextFieldMapperTests.java | 13 +++++ .../PatternedTextIntegrationTests.java | 49 ++++++++++++++----- 4 files changed, 85 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java index 58f7a1955731b..a080b371532e9 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.logsdb.patternedtext; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; @@ -45,6 +47,17 @@ protected String getTestRestCluster() { return cluster.getHttpAddresses(); } + @ParametersFactory(argumentFormatting = "disableTemplating=%b") + public static List args() { + return List.of(new Object[] { true }, new Object[] { false }); + } + + private final boolean disableTemplating; + + public PatternedTextBasicRestIT(boolean disableTemplating) { + this.disableTemplating = disableTemplating; + } + @SuppressWarnings("unchecked") public void testBulkInsertThenMatchAllSource() throws IOException { @@ -63,11 +76,12 @@ public void testBulkInsertThenMatchAllSource() throws IOException { "type": "date" }, "message": { - "type": "patterned_text" + "type": "patterned_text", + "disable_templating": %disable_templating% } } } - """; + """.replace("%disable_templating%", Boolean.toString(disableTemplating)); String indexName = "test-index"; createIndex(indexName, settings.build(), mapping); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index a4451b9cb58e4..2a59faf058a63 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockStoredFieldsReader; import org.elasticsearch.index.mapper.SourceValueFetcher; @@ -50,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; public class PatternedTextFieldType extends StringFieldType { @@ -119,6 +121,10 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) private IOFunction, IOException>> getValueFetcherProvider( SearchExecutionContext searchExecutionContext ) { + if (disableTemplating) { + return storedFieldFetcher(storedNamed()); + } + return context -> { ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null); SourceProvider sourceProvider = searchExecutionContext.lookup(); @@ -133,6 +139,18 @@ private IOFunction, IOExcepti }; } + private static IOFunction, IOException>> storedFieldFetcher(String name) { + var loader = StoredFieldLoader.create(false, Set.of(name)); + return context -> { + var leafLoader = loader.getLoader(context, null); + return docId -> { + leafLoader.advanceTo(docId); + var storedFields = leafLoader.storedFields(); + return storedFields.get(name); + }; + }; + } + private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext context) { // Disable scoring similarly to match_only_text if (hasPositions) { diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java index 382afbbfaebbe..cb0543e547475 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java @@ -97,6 +97,16 @@ public void testPhraseQuerySyntheticSource() throws IOException { assertPhraseQuery(createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text")))); } + public void testPhraseQueryStandardSourceDisableTemplating() throws IOException { + assertPhraseQuery(createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true)))); + } + + public void testPhraseQuerySyntheticSourceDisableTemplating() throws IOException { + assertPhraseQuery( + createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true))) + ); + } + private void assertPhraseQuery(MapperService mapperService) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); @@ -322,6 +332,9 @@ private Tuple generateValue() { private void mapping(XContentBuilder b) throws IOException { b.field("type", "patterned_text"); + if (randomBoolean()) { + b.field("disable_templating", true); + } } @Override diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java index 708dbec89d048..00f7fdcab24fd 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.logsdb.patternedtext; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.DocWriteRequest; @@ -41,6 +43,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -56,6 +59,27 @@ public class PatternedTextIntegrationTests extends ESSingleNodeTestCase { private static final Logger logger = LogManager.getLogger(PatternedTextIntegrationTests.class); + @ParametersFactory(argumentFormatting = "indexOptions=%s, disableTemplating=%b") + public static List args() { + List args = new ArrayList<>(); + for (var indexOption : new String[] { "docs", "positions" }) { + for (var templating : new boolean[] { true, false }) { + args.add(new Object[] { indexOption, templating }); + } + } + return Collections.unmodifiableList(args); + } + + private final String indexOptions; + private final boolean disableTemplating; + private final String mapping; + + public PatternedTextIntegrationTests(String indexOptions, boolean disableTemplating) { + this.indexOptions = indexOptions; + this.disableTemplating = disableTemplating; + this.mapping = getMapping(indexOptions, disableTemplating); + } + @Override protected Settings nodeSettings() { return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build(); @@ -75,17 +99,15 @@ protected Collection> getPlugins() { "@timestamp": { "type": "date" }, "field_match_only_text": { "type": "match_only_text" }, "field_patterned_text": { - "type": "patterned_text", - "index_options": "%", - "analyzer": "standard" + "type": "patterned_text", + "index_options": "%index_options%", + "disable_templating": "%disable_templating%", + "analyzer": "standard" } } } """; - private static final String MAPPING_DOCS_ONLY = MAPPING_TEMPLATE.replace("%", "docs"); - private static final String MAPPING_POSITIONS = MAPPING_TEMPLATE.replace("%", "positions"); - private static final Settings LOGSDB_SETTING = Settings.builder().put(IndexSettings.MODE.getKey(), "logsdb").build(); @Before @@ -100,8 +122,12 @@ public void cleanup() { } } + private String getMapping(String indexOptions, boolean disableTemplating) { + return MAPPING_TEMPLATE.replace("%index_options%", indexOptions) + .replace("%disable_templating%", Boolean.toString(disableTemplating)); + } + public void testSourceMatchAllManyValues() throws IOException { - var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS; var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping); createIndex(INDEX, createRequest); @@ -114,7 +140,6 @@ public void testSourceMatchAllManyValues() throws IOException { } public void testLargeValueIsStored() throws IOException { - var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS; var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping); IndexService indexService = createIndex(INDEX, createRequest); @@ -136,7 +161,6 @@ public void testLargeValueIsStored() throws IOException { } public void testSmallValueNotStored() throws IOException { - var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS; var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping); IndexService indexService = createIndex(INDEX, createRequest); @@ -152,13 +176,16 @@ public void testSmallValueNotStored() throws IOException { try (var searcher = indexService.getShard(0).acquireSearcher(INDEX)) { try (var indexReader = searcher.getIndexReader()) { var document = indexReader.storedFields().document(0); - assertNull(document.getField("field_patterned_text.stored")); + if (disableTemplating) { + assertEquals(document.getField("field_patterned_text.stored").binaryValue().utf8ToString(), message); + } else { + assertNull(document.getField("field_patterned_text.stored")); + } } } } public void testQueryResultsSameAsMatchOnlyText() throws IOException { - var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS; var createRequest = new CreateIndexRequest(INDEX).mapping(mapping); if (randomBoolean()) { From 5c5377f3182fc4a9a540c536343898e48e29906d Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 16 Sep 2025 13:46:35 -0700 Subject: [PATCH 3/3] Update comment in PatternedTextIntegrationTests#testSmallValueNotStored --- .../logsdb/patternedtext/PatternedTextIntegrationTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java index 00f7fdcab24fd..1801ac7ffdf55 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java @@ -172,7 +172,7 @@ public void testSmallValueNotStored() throws IOException { assertMappings(); assertMessagesInSource(messages); - // assert does not contain stored field + // assert only contains stored field if templating is disabled try (var searcher = indexService.getShard(0).acquireSearcher(INDEX)) { try (var indexReader = searcher.getIndexReader()) { var document = indexReader.storedFields().document(0);