Skip to content

Patterned text conditionally disable templating #134744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jordan-powers merged 4 commits into from
Sep 17, 2025
Merged

Patterned text conditionally disable templating #134744

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
82716e4
Conditionally disable templating in patterned_text
jordan-powers Sep 15, 2025
daec41c
Add patterned_text tests for disable_templating:true
jordan-powers Sep 15, 2025
5c5377f
Update comment in PatternedTextIntegrationTests#testSmallValueNotStored
jordan-powers Sep 16, 2025
be850f2
Merge remote-tracking branch 'upstream/main' into patterned-text-cond…
jordan-powers Sep 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions ...aRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

package org.elasticsearch.xpack.logsdb.patternedtext;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -45,6 +47,17 @@ protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}

@ParametersFactory(argumentFormatting = "disableTemplating=%b")
public static List<Object[]> args() {
return List.of(new Object[] { true }, new Object[] { false });
}

private final boolean disableTemplating;

public PatternedTextBasicRestIT(boolean disableTemplating) {
this.disableTemplating = disableTemplating;
}

@SuppressWarnings("unchecked")
public void testBulkInsertThenMatchAllSource() throws IOException {

Expand All @@ -63,11 +76,12 @@ public void testBulkInsertThenMatchAllSource() throws IOException {
"type": "date"
},
"message": {
"type": "patterned_text"
"type": "patterned_text",
"disable_templating": %disable_templating%
}
}
}
""";
""".replace("%disable_templating%", Boolean.toString(disableTemplating));

String indexName = "test-index";
createIndex(indexName, settings.build(), mapping);
Expand Down
41 changes: 30 additions & 11 deletions .../src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@
import org.elasticsearch.index.mapper.MapperBuilderContext;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MappingParserContext;
import org.elasticsearch.index.mapper.SourceLoader;
import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
import org.elasticsearch.index.mapper.TextParams;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.xcontent.XContentBuilder;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -261,12 +264,17 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
throw new IllegalArgumentException("Multiple values are not allowed for field [" + fieldType().name() + "].");
}

// Parse template and args
PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);

// Add index on original value
context.doc().add(new Field(fieldType().name(), value, fieldType));

if (fieldType().disableTemplating()) {
context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value)));
return;
}

// Parse template and args
PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);

// Add template_id doc_values
context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId())));

Expand Down Expand Up @@ -305,14 +313,25 @@ interface DocValuesSupplier {

@Override
protected SyntheticSourceSupport syntheticSourceSupport() {
return new SyntheticSourceSupport.Native(
() -> new CompositeSyntheticFieldLoader(
leafName(),
fullPath(),
new PatternedTextSyntheticFieldLoaderLayer(
fieldType().name(),
leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType())
)
return new SyntheticSourceSupport.Native(this::getSyntheticFieldLoader);
}

private SourceLoader.SyntheticFieldLoader getSyntheticFieldLoader() {
if (fieldType().disableTemplating()) {
return new StringStoredFieldFieldLoader(fieldType().storedNamed(), fieldType().name(), leafName()) {
@Override
protected void write(XContentBuilder b, Object value) throws IOException {
b.value(((BytesRef) value).utf8ToString());
}
};
}

return new CompositeSyntheticFieldLoader(
leafName(),
fullPath(),
new PatternedTextSyntheticFieldLoaderLayer(
fieldType().name(),
leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType())
)
);
}
Expand Down
23 changes: 23 additions & 0 deletions ...db/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData;
import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
import org.elasticsearch.index.mapper.BlockLoader;
import org.elasticsearch.index.mapper.BlockStoredFieldsReader;
import org.elasticsearch.index.mapper.SourceValueFetcher;
import org.elasticsearch.index.mapper.StringFieldType;
import org.elasticsearch.index.mapper.TextFieldMapper;
Expand All @@ -49,6 +51,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

public class PatternedTextFieldType extends StringFieldType {

Expand Down Expand Up @@ -118,6 +121,10 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> getValueFetcherProvider(
SearchExecutionContext searchExecutionContext
) {
if (disableTemplating) {
return storedFieldFetcher(storedNamed());
}

return context -> {
ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null);
SourceProvider sourceProvider = searchExecutionContext.lookup();
Expand All @@ -132,6 +139,18 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
};
}

private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> storedFieldFetcher(String name) {
var loader = StoredFieldLoader.create(false, Set.of(name));
return context -> {
var leafLoader = loader.getLoader(context, null);
return docId -> {
leafLoader.advanceTo(docId);
var storedFields = leafLoader.storedFields();
return storedFields.get(name);
};
};
}

private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext context) {
// Disable scoring similarly to match_only_text
if (hasPositions) {
Expand Down Expand Up @@ -262,6 +281,10 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions,

@Override
public BlockLoader blockLoader(BlockLoaderContext blContext) {
if (disableTemplating) {
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedNamed());
}

return new PatternedTextBlockLoader((leafReader -> PatternedTextCompositeValues.from(leafReader, this)));
}

Expand Down
13 changes: 13 additions & 0 deletions ...test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,16 @@ public void testPhraseQuerySyntheticSource() throws IOException {
assertPhraseQuery(createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text"))));
}

public void testPhraseQueryStandardSourceDisableTemplating() throws IOException {
assertPhraseQuery(createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true))));
}

public void testPhraseQuerySyntheticSourceDisableTemplating() throws IOException {
assertPhraseQuery(
createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true)))
);
}

private void assertPhraseQuery(MapperService mapperService) throws IOException {
try (Directory directory = newDirectory()) {
RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
Expand Down Expand Up @@ -322,6 +332,9 @@ private Tuple<String, String> generateValue() {

private void mapping(XContentBuilder b) throws IOException {
b.field("type", "patterned_text");
if (randomBoolean()) {
b.field("disable_templating", true);
}
}

@Override
Expand Down
51 changes: 39 additions & 12 deletions ...test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

package org.elasticsearch.xpack.logsdb.patternedtext;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.DocWriteRequest;
Expand Down Expand Up @@ -41,6 +43,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand All @@ -56,6 +59,27 @@
public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
private static final Logger logger = LogManager.getLogger(PatternedTextIntegrationTests.class);

@ParametersFactory(argumentFormatting = "indexOptions=%s, disableTemplating=%b")
public static List<Object[]> args() {
List<Object[]> args = new ArrayList<>();
for (var indexOption : new String[] { "docs", "positions" }) {
for (var templating : new boolean[] { true, false }) {
args.add(new Object[] { indexOption, templating });
}
}
return Collections.unmodifiableList(args);
}

private final String indexOptions;
private final boolean disableTemplating;
private final String mapping;

public PatternedTextIntegrationTests(String indexOptions, boolean disableTemplating) {
this.indexOptions = indexOptions;
this.disableTemplating = disableTemplating;
this.mapping = getMapping(indexOptions, disableTemplating);
}

@Override
protected Settings nodeSettings() {
return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build();
Expand All @@ -75,17 +99,15 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
"@timestamp": { "type": "date" },
"field_match_only_text": { "type": "match_only_text" },
"field_patterned_text": {
"type": "patterned_text",
"index_options": "%",
"analyzer": "standard"
"type": "patterned_text",
"index_options": "%index_options%",
"disable_templating": "%disable_templating%",
"analyzer": "standard"
}
}
}
""";

private static final String MAPPING_DOCS_ONLY = MAPPING_TEMPLATE.replace("%", "docs");
private static final String MAPPING_POSITIONS = MAPPING_TEMPLATE.replace("%", "positions");

private static final Settings LOGSDB_SETTING = Settings.builder().put(IndexSettings.MODE.getKey(), "logsdb").build();

@Before
Expand All @@ -100,8 +122,12 @@ public void cleanup() {
}
}

private String getMapping(String indexOptions, boolean disableTemplating) {
return MAPPING_TEMPLATE.replace("%index_options%", indexOptions)
.replace("%disable_templating%", Boolean.toString(disableTemplating));
}

public void testSourceMatchAllManyValues() throws IOException {
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
createIndex(INDEX, createRequest);

Expand All @@ -114,7 +140,6 @@ public void testSourceMatchAllManyValues() throws IOException {
}

public void testLargeValueIsStored() throws IOException {
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
IndexService indexService = createIndex(INDEX, createRequest);

Expand All @@ -136,7 +161,6 @@ public void testLargeValueIsStored() throws IOException {
}

public void testSmallValueNotStored() throws IOException {
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
IndexService indexService = createIndex(INDEX, createRequest);

Expand All @@ -148,17 +172,20 @@ public void testSmallValueNotStored() throws IOException {
assertMappings();
assertMessagesInSource(messages);

// assert does not contain stored field
// assert only contains stored field if templating is disabled
try (var searcher = indexService.getShard(0).acquireSearcher(INDEX)) {
try (var indexReader = searcher.getIndexReader()) {
var document = indexReader.storedFields().document(0);
assertNull(document.getField("field_patterned_text.stored"));
Copy link
Contributor

@parkertimmins parkertimmins Sep 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: ^ maybe update the above comment to "assert only contains stored field if templating is disabled"

if (disableTemplating) {
assertEquals(document.getField("field_patterned_text.stored").binaryValue().utf8ToString(), message);
} else {
assertNull(document.getField("field_patterned_text.stored"));
}
}
}
}

public void testQueryResultsSameAsMatchOnlyText() throws IOException {
var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
var createRequest = new CreateIndexRequest(INDEX).mapping(mapping);

if (randomBoolean()) {
Expand Down