Store raw pattern text fields in binary doc values

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -218,6 +218,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion FALLBACK_TEXT_FIELDS_BINARY_DOC_VALUES_FORMAT_CHECK = def(9_065_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion READ_SI_FILES_FROM_MEMORY_FOR_HOLLOW_COMMITS = def(9_066_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion FLATTENED_FIELD_TSDB_CODEC_USE_BINARY_DOC_VALUES = def(9_067_0_00, Version.LUCENE_10_3_2);
+    public static final IndexVersion STORE_PATTERN_TEXT_FIELDS_IN_BINARY_DOC_VALUES = def(9_068_0_00, Version.LUCENE_10_3_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/AbstractStringTypeLogsdbRollingUpgradeTestCase.java

@@ -47,13 +47,22 @@ public AbstractStringTypeLogsdbRollingUpgradeTestCase(String dataStreamName, Str
 
     @Before
     public void createIndex() throws Exception {
+        checkRequiredFeatures();
         LogsdbIndexingRollingUpgradeIT.maybeEnableLogsdbByDefault();
 
         // data stream name should already be reflective of whats being tested, so template id can be random
         templateId = UUID.randomUUID().toString();
         LogsdbIndexingRollingUpgradeIT.createTemplate(dataStreamName, templateId, template);
     }
 
+    /**
+     * Override this method to add feature checks that must pass before the test runs.
+     * Use {@code assumeTrue} to skip the test if required features are not available.
+     */
+    protected void checkRequiredFeatures() throws Exception {
+        // Default: no additional feature requirements
+    }
+
     protected List<String> getMessages() {
         return messages;
     }

x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/PatternTextRollingUpgradeIT.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import java.util.Arrays;
+
+public class PatternTextRollingUpgradeIT extends AbstractStringTypeLogsdbRollingUpgradeTestCase {
+
+    private static final String MIN_VERSION = "gte_v9.2.0";
+    private static final String DATA_STREAM_NAME_PREFIX = "logs-pattern-text-bwc-test";
+
+    private static final String TEMPLATE = """
+        {
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "length": {
+                  "type": "long"
+                },
+                "factor": {
+                  "type": "double"
+                },
+                "message": {
+                  "type": "pattern_text"
+                }
+              }
+            }
+        }""";
+
+    private static final String TEMPLATE_WITH_MULTI_FIELD = """
+        {
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "length": {
+                  "type": "long"
+                },
+                "factor": {
+                  "type": "double"
+                },
+                "message": {
+                  "type": "pattern_text",
+                  "fields": {
+                    "kwd": {
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            }
+        }""";
+
+    private static final String TEMPLATE_WITH_MULTI_FIELD_AND_IGNORE_ABOVE = """
+        {
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "length": {
+                  "type": "long"
+                },
+                "factor": {
+                  "type": "double"
+                },
+                "message": {
+                  "type": "pattern_text",
+                  "fields": {
+                    "kwd": {
+                      "type": "keyword",
+                      "ignore_above": 50
+                    }
+                  }
+                }
+              }
+            }
+        }""";
+
+    public PatternTextRollingUpgradeIT(String template, String testScenario) {
+        super(DATA_STREAM_NAME_PREFIX + "." + testScenario, template);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> data() {
+        return Arrays.asList(
+            new Object[][] {
+                { TEMPLATE, "basic" },
+                { TEMPLATE_WITH_MULTI_FIELD, "with-keyword-multi-field" },
+                { TEMPLATE_WITH_MULTI_FIELD_AND_IGNORE_ABOVE, "with-keyword-multi-field-and-ignore-above" } }
+        );
+    }
+
+    @Override
+    protected void checkRequiredFeatures() {
+        assumeTrue("pattern_text only available from 9.2.0 onward", oldClusterHasFeature(MIN_VERSION));
+    }
+
+}

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextCompositeValues.java

@@ -20,28 +20,31 @@
 import java.util.Set;
 
 /**
- * If there are values which exceed 32kb, they cannot be stored as doc values
- * and must be in a stored field. This class combines the doc values with the
- * larges values which are in stored fields. Despite being backed by stored
- * fields, this class implements a doc value interface.
+ * Values which exceeds 32kb cannot be stored as sorted set doc values. Such values must be stored separately in binary doc values, which
+ * do not have length limitations. This class combines the regular doc values with the raw values from binary doc values.
  */
 public final class PatternTextCompositeValues extends BinaryDocValues {
+
     private final LeafStoredFieldLoader storedTemplateLoader;
     private final String storedMessageFieldName;
     private final BinaryDocValues patternTextDocValues;
     private final SortedSetDocValues templateIdDocValues;
+    private final BinaryDocValues rawTextDocValues;
     private boolean hasDocValue = false;
+    private boolean hasRawTextDocValue = false;
 
     PatternTextCompositeValues(
         LeafStoredFieldLoader storedTemplateLoader,
         String storedMessageFieldName,
         BinaryDocValues patternTextDocValues,
-        SortedSetDocValues templateIdDocValues
+        SortedSetDocValues templateIdDocValues,
+        BinaryDocValues rawTextDocValues
     ) {
         this.storedTemplateLoader = storedTemplateLoader;
         this.storedMessageFieldName = storedMessageFieldName;
         this.patternTextDocValues = patternTextDocValues;
         this.templateIdDocValues = templateIdDocValues;
+        this.rawTextDocValues = rawTextDocValues;
     }
 
     static PatternTextCompositeValues from(LeafReader leafReader, PatternTextFieldType fieldType) throws IOException {
@@ -57,17 +60,40 @@ static PatternTextCompositeValues from(LeafReader leafReader, PatternTextFieldTy
             fieldType.argsInfoFieldName(),
             fieldType.useBinaryDocValuesArgs()
         );
+
+        // load binary doc values (for newer indices that store raw values in binary doc values)
+        BinaryDocValues rawBinaryDocValues = leafReader.getBinaryDocValues(fieldType.storedNamed());
+        if (rawBinaryDocValues == null) {
+            // use an empty object here to avoid making null checks later
+            rawBinaryDocValues = DocValues.emptyBinary();
+        }
+
+        // load stored field loader (for older indices that store raw values in stored fields)
         StoredFieldLoader storedFieldLoader = StoredFieldLoader.create(false, Set.of(fieldType.storedNamed()));
         LeafStoredFieldLoader storedTemplateLoader = storedFieldLoader.getLoader(leafReader.getContext(), null);
-        return new PatternTextCompositeValues(storedTemplateLoader, fieldType.storedNamed(), docValues, templateIdDocValues);
+
+        return new PatternTextCompositeValues(
+            storedTemplateLoader,
+            fieldType.storedNamed(),
+            docValues,
+            templateIdDocValues,
+            rawBinaryDocValues
+        );
     }
 
     public BytesRef binaryValue() throws IOException {
         if (hasDocValue) {
             return patternTextDocValues.binaryValue();
         }
 
-        // If there is no doc value, the value was too large and was put in a stored field
+        // if there is no doc value, then the value was too large to be analyzed or templating was disabled
+
+        // for newer indices, the value is stored in binary doc values
+        if (hasRawTextDocValue) {
+            return rawTextDocValues.binaryValue();
+        }
+
+        // for older indices, it's stored in stored fields
         var storedFields = storedTemplateLoader.storedFields();
         List<Object> storedValues = storedFields.get(storedMessageFieldName);
         assert storedValues != null && storedValues.size() == 1 && storedValues.getFirst() instanceof BytesRef;
@@ -81,7 +107,8 @@ public int docID() {
     public boolean advanceExact(int i) throws IOException {
         boolean hasValue = templateIdDocValues.advanceExact(i);
         hasDocValue = patternTextDocValues.advanceExact(i);
-        if (hasValue && hasDocValue == false) {
+        hasRawTextDocValue = rawTextDocValues.advanceExact(i);
+        if (hasValue && hasDocValue == false && hasRawTextDocValue == false) {
             storedTemplateLoader.advanceTo(i);
         }
         return hasValue;
@@ -100,6 +127,6 @@ public int advance(int i) {
 
     @Override
     public long cost() {
-        return templateIdDocValues.cost() + patternTextDocValues.cost();
+        return templateIdDocValues.cost() + patternTextDocValues.cost() + rawTextDocValues.cost();
     }
 }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapper.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.BinaryDocValuesSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -97,17 +98,31 @@ public static class Builder extends TextFamilyBuilder {
         private final Parameter<NamedAnalyzer> analyzer;
         private final Parameter<Boolean> disableTemplating;
         private final IndexVersion indexCreatedVersion;
+        private final boolean useBinaryDocValuesForRawText;
 
         public Builder(String name, MappingParserContext context) {
-            this(name, context.indexVersionCreated(), context.getIndexSettings(), context.isWithinMultiField());
+            this(
+                name,
+                context.indexVersionCreated(),
+                context.getIndexSettings(),
+                context.isWithinMultiField(),
+                useBinaryDocValuesForRawText(context.getIndexSettings())
+            );
         }
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings indexSettings, boolean isWithinMultiField) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            IndexSettings indexSettings,
+            boolean isWithinMultiField,
+            boolean useBinaryDocValuesForRawText
+        ) {
             super(name, indexCreatedVersion, isWithinMultiField);
             this.indexSettings = indexSettings;
             this.analyzer = analyzerParam(name, m -> ((PatternTextFieldMapper) m).analyzer);
             this.disableTemplating = disableTemplatingParameter(indexSettings);
             this.indexCreatedVersion = indexCreatedVersion;
+            this.useBinaryDocValuesForRawText = useBinaryDocValuesForRawText;
         }
 
         private boolean useBinaryDocValuesForArgsColumn() {
@@ -130,7 +145,8 @@ private PatternTextFieldType buildFieldType(FieldType fieldType, MapperBuilderCo
                 meta.getValue(),
                 context.isSourceSynthetic(),
                 isWithinMultiField(),
-                useBinaryDocValuesForArgsColumn()
+                useBinaryDocValuesForArgsColumn(),
+                useBinaryDocValuesForRawText
             );
         }
 
@@ -217,6 +233,7 @@ public PatternTextFieldMapper build(MapperBuilderContext context) {
     private final FieldType fieldType;
     private final KeywordFieldMapper templateIdMapper;
     private final boolean useBinaryDocValueArgs;
+    private final boolean useBinaryDocValuesForRawText;
 
     private PatternTextFieldMapper(
         String simpleName,
@@ -236,6 +253,7 @@ private PatternTextFieldMapper(
         this.indexOptions = builder.indexOptions.getValue();
         this.templateIdMapper = templateIdMapper;
         this.useBinaryDocValueArgs = builder.useBinaryDocValuesForArgsColumn();
+        this.useBinaryDocValuesForRawText = builder.useBinaryDocValuesForRawText;
     }
 
     @Override
@@ -245,7 +263,8 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexSettings, fieldType().isWithinMultiField()).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexSettings, fieldType().isWithinMultiField(), useBinaryDocValuesForRawText)
+            .init(this);
     }
 
     @Override
@@ -275,7 +294,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         context.doc().add(new Field(fieldType().name(), value, fieldType));
 
         if (fieldType().disableTemplating()) {
-            context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value)));
+            storePatternAsRawText(context, value);
             return;
         }
 
@@ -285,8 +304,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         // Add template_id doc_values
         context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId())));
 
-        if (parts.useStoredField()) {
-            context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value)));
+        if (parts.useBinaryDocValuesForRawText()) {
+            storePatternAsRawText(context, value);
         } else {
             // Add template doc_values
             context.doc().add(new SortedSetDocValuesField(fieldType().templateFieldName(), new BytesRef(parts.template())));
@@ -307,6 +326,27 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         }
     }
 
+    /**
+     * Store the value as a raw text field, without analyzing it. This can happen when templating is disabled or when the value is too long
+     * to be analyzed.
+     *
+     * Values may be stored in binary doc values or in stored fields, both of which don't have the same length limitations as regular doc
+     * values do.
+     */
+    private void storePatternAsRawText(DocumentParserContext context, final String value) {
+        if (useBinaryDocValuesForRawText) {
+            context.doc().add(new BinaryDocValuesField(fieldType().storedNamed(), new BytesRef(value)));
+        } else {
+            // for bwc, store in stored fields
+            context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value)));
+        }
+    }
+
+    private static boolean useBinaryDocValuesForRawText(IndexSettings indexSettings) {
+        return indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STORE_PATTERN_TEXT_FIELDS_IN_BINARY_DOC_VALUES)
+            && indexSettings.useTimeSeriesDocValuesFormat();
+    }
+
     @Override
     protected String contentType() {
         return PatternTextFieldType.CONTENT_TYPE;
@@ -329,6 +369,17 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
 
     private SourceLoader.SyntheticFieldLoader getSyntheticFieldLoader() {
         if (fieldType().disableTemplating()) {
+            if (useBinaryDocValuesForRawText) {
+                return new BinaryDocValuesSyntheticFieldLoader(fieldType().storedNamed()) {
+                    @Override
+                    protected void writeValue(XContentBuilder b, BytesRef value) throws IOException {
+                        // pattern text fields are not multi-valued, so there is no special encoding here unlike other fields that use
+                        // binary doc values. As a result, we don't need to much and this function remains simple
+                        b.field(leafName(), value.utf8ToString());
+                    }
+                };
+            }
+
             return new StringStoredFieldFieldLoader(fieldType().storedNamed(), fieldType().name(), leafName()) {
                 @Override
                 protected void write(XContentBuilder b, Object value) throws IOException {