Store flattened field data in binary doc values

docs/changelog/140246.yaml

@@ -0,0 +1,5 @@
+pr: 140246
+summary: Store flattened field data in binary doc values
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -217,6 +217,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion MOD_ROUTING_FUNCTION = def(9_064_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion FALLBACK_TEXT_FIELDS_BINARY_DOC_VALUES_FORMAT_CHECK = def(9_065_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion READ_SI_FILES_FROM_MEMORY_FOR_HOLLOW_COMMITS = def(9_066_0_00, Version.LUCENE_10_3_2);
+    public static final IndexVersion FLATTENED_FIELD_TSDB_CODEC_USE_BINARY_DOC_VALUES = def(9_067_0_00, Version.LUCENE_10_3_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/fielddata/MultiValuedSortedBinaryDocValues.java

@@ -49,9 +49,15 @@ public static MultiValuedSortedBinaryDocValues from(LeafReader leafReader, Strin
         if (counts == null) {
             return new IntegratedCounts(values);
         } else {
+            Sparsity sparsity = Sparsity.UNKNOWN;
+            ValueMode valueMode = ValueMode.UNKNOWN;
+
             DocValuesSkipper countsSkipper = leafReader.getDocValuesSkipper(countsFieldName);
-            Sparsity sparsity = countsSkipper.docCount() == leafReader.maxDoc() ? Sparsity.DENSE : Sparsity.SPARSE;
-            ValueMode valueMode = countsSkipper.maxValue() == 1 ? ValueMode.SINGLE_VALUED : ValueMode.MULTI_VALUED;
+            if (countsSkipper != null) {
+                sparsity = countsSkipper.docCount() == leafReader.maxDoc() ? Sparsity.DENSE : Sparsity.SPARSE;
+                valueMode = countsSkipper.maxValue() == 1 ? ValueMode.SINGLE_VALUED : ValueMode.MULTI_VALUED;
+            }
+
             return new SeparateCounts(values, counts, sparsity, valueMode);
         }
     }

server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java

@@ -202,7 +202,7 @@ public NumericDocValues getNormValues(String field) throws IOException {
 
     @Override
     public DocValuesSkipper getDocValuesSkipper(String s) throws IOException {
-        throw new UnsupportedOperationException();
+        return null;
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -16,7 +16,6 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.InvertableType;
-import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DocValuesSkipIndexType;
@@ -112,7 +111,6 @@
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
 import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 import static org.elasticsearch.index.mapper.FieldMapper.Parameter.useTimeSeriesDocValuesSkippers;
-import static org.elasticsearch.index.mapper.MultiValuedBinaryDocValuesField.SeparateCount.COUNT_FIELD_SUFFIX;
 
 /**
  * A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -345,6 +343,10 @@ public DocValuesParameter.Values docValuesParameters() {
             return docValuesParameters.getValue();
         }
 
+        boolean usesBinaryDocValues() {
+            return docValuesParameters().enabled() && docValuesParameters().cardinality() == DocValuesParameter.Values.Cardinality.HIGH;
+        }
+
         public SimilarityProvider similarity() {
             return this.similarity.get();
         }
@@ -544,7 +546,7 @@ public static final class KeywordFieldType extends TextFamilyFieldType {
         private final boolean eagerGlobalOrdinals;
         private final FieldValues<String> scriptValues;
         private final boolean isDimension;
-        private final DocValuesParameter.Values docValuesParameters;
+        private final boolean usesBinaryDocValues;
 
         public KeywordFieldType(
             String name,
@@ -573,22 +575,32 @@ public KeywordFieldType(
             this.nullValue = builder.nullValue.getValue();
             this.scriptValues = builder.scriptValues();
             this.isDimension = builder.dimension.getValue();
-            this.docValuesParameters = builder.docValuesParameters();
+            this.usesBinaryDocValues = builder.usesBinaryDocValues();
         }
 
         public KeywordFieldType(String name) {
-            this(name, true, true, Collections.emptyMap());
+            this(name, true, true, false, Collections.emptyMap());
         }
 
         public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
+            this(name, isIndexed, hasDocValues, false, meta);
+        }
+
+        public KeywordFieldType(
+            String name,
+            boolean isIndexed,
+            boolean hasDocValues,
+            boolean usesBinaryDocValues,
+            Map<String, String> meta
+        ) {
             super(name, IndexType.terms(isIndexed, hasDocValues), false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta, false, false);
             this.normalizer = Lucene.KEYWORD_ANALYZER;
             this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
             this.nullValue = null;
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
-            this.docValuesParameters = DEFAULT_DOC_VALUES_PARAMS;
+            this.usesBinaryDocValues = usesBinaryDocValues;
         }
 
         public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSource) {
@@ -607,7 +619,7 @@ public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSou
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
-            this.docValuesParameters = DEFAULT_DOC_VALUES_PARAMS;
+            this.usesBinaryDocValues = false;
         }
 
         public KeywordFieldType(String name, NamedAnalyzer analyzer) {
@@ -626,7 +638,11 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) {
             this.eagerGlobalOrdinals = false;
             this.scriptValues = null;
             this.isDimension = false;
-            this.docValuesParameters = DEFAULT_DOC_VALUES_PARAMS;
+            this.usesBinaryDocValues = false;
+        }
+
+        public boolean usesBinaryDocValues() {
+            return usesBinaryDocValues;
         }
 
         @Override
@@ -639,7 +655,7 @@ public Query termQuery(Object value, SearchExecutionContext context) {
             failIfNotIndexedNorDocValuesFallback(context);
             if (indexType.hasTerms()) {
                 return super.termQuery(value, context);
-            } else if (storedInBinaryDocValues()) {
+            } else if (usesBinaryDocValues) {
                 return new SlowCustomBinaryDocValuesTermQuery(name(), indexedValueForSearch(value));
             } else {
                 return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
@@ -651,7 +667,7 @@ public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
             failIfNotIndexedNorDocValuesFallback(context);
             if (indexType.hasTerms()) {
                 return super.termsQuery(values, context);
-            } else if (storedInBinaryDocValues()) {
+            } else if (usesBinaryDocValues) {
                 return new StringScriptFieldTermsQuery(
                     new Script(""),
                     ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -675,7 +691,7 @@ public Query rangeQuery(
             failIfNotIndexedNorDocValuesFallback(context);
             if (indexType.hasTerms()) {
                 return super.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context);
-            } else if (storedInBinaryDocValues()) {
+            } else if (usesBinaryDocValues) {
                 return new StringScriptFieldRangeQuery(
                     new Script(""),
                     ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -709,7 +725,7 @@ public Query fuzzyQuery(
             failIfNotIndexedNorDocValuesFallback(context);
             if (indexType.hasTerms()) {
                 return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context, rewriteMethod);
-            } else if (storedInBinaryDocValues()) {
+            } else if (usesBinaryDocValues) {
                 return StringScriptFieldFuzzyQuery.build(
                     new Script(""),
                     ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -741,7 +757,7 @@ public Query prefixQuery(
             failIfNotIndexedNorDocValuesFallback(context);
             if (indexType.hasTerms()) {
                 return super.prefixQuery(value, method, caseInsensitive, context);
-            } else if (storedInBinaryDocValues()) {
+            } else if (usesBinaryDocValues) {
                 return new StringScriptFieldPrefixQuery(
                     new Script(""),
                     ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -769,7 +785,7 @@ public Query termQueryCaseInsensitive(Object value, SearchExecutionContext conte
             failIfNotIndexedNorDocValuesFallback(context);
             if (indexType.hasTerms()) {
                 return super.termQueryCaseInsensitive(value, context);
-            } else if (storedInBinaryDocValues()) {
+            } else if (usesBinaryDocValues) {
                 return new StringScriptFieldTermQuery(
                     new Script(""),
                     ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -794,7 +810,7 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi
             if (indexType.hasTerms()) {
                 terms = MultiTerms.getTerms(reader, name());
             } else if (hasDocValues()) {
-                if (storedInBinaryDocValues()) {
+                if (usesBinaryDocValues) {
                     throw new UnsupportedOperationException("TODO");
                 } else {
                     terms = SortedSetDocValuesTerms.getTerms(reader, name());
@@ -828,10 +844,6 @@ public String typeName() {
             return CONTENT_TYPE;
         }
 
-        public boolean storedInBinaryDocValues() {
-            return docValuesParameters.enabled() && docValuesParameters.cardinality() == DocValuesParameter.Values.Cardinality.HIGH;
-        }
-
         @Override
         public boolean eagerGlobalOrdinals() {
             return eagerGlobalOrdinals;
@@ -846,7 +858,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSourceEnabled())) {
                 BlockLoaderFunctionConfig cfg = blContext.blockLoaderFunctionConfig();
                 if (cfg == null) {
-                    if (storedInBinaryDocValues()) {
+                    if (usesBinaryDocValues) {
                         return new BytesRefsFromBinaryMultiSeparateCountBlockLoader(name());
                     } else {
                         return new BytesRefsFromOrdsBlockLoader(name());
@@ -986,7 +998,7 @@ protected BytesRef storedToBytesRef(Object stored) {
         }
 
         private IndexFieldData.Builder fieldDataFromDocValues() {
-            if (storedInBinaryDocValues()) {
+            if (usesBinaryDocValues) {
                 return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, KeywordDocValuesField::new);
             } else {
                 return new SortedSetOrdinalsIndexFieldData.Builder(
@@ -1076,7 +1088,7 @@ public Query wildcardQuery(
                     value = indexedValueForSearch(value).utf8ToString();
                 }
 
-                if (storedInBinaryDocValues()) {
+                if (usesBinaryDocValues) {
                     return new SlowCustomBinaryDocValuesWildcardQuery(name(), value, caseInsensitive);
                 }
 
@@ -1102,7 +1114,7 @@ public Query normalizedWildcardQuery(String value, MultiTermQuery.RewriteMethod
                     value = indexedValueForSearch(value).utf8ToString();
                 }
 
-                if (storedInBinaryDocValues()) {
+                if (usesBinaryDocValues) {
                     return new StringScriptFieldWildcardQuery(
                         new Script(""),
                         ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -1134,7 +1146,7 @@ public Query regexpQuery(
                     throw new IllegalArgumentException("Match flags not yet implemented [" + matchFlags + "]");
                 }
 
-                if (storedInBinaryDocValues()) {
+                if (usesBinaryDocValues) {
                     return new StringScriptFieldRegexpQuery(
                         new Script(""),
                         ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -1365,20 +1377,13 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
             throw new IllegalArgumentException(msg);
         }
 
-        if (fieldType().storedInBinaryDocValues()) {
+        if (fieldType().usesBinaryDocValues()) {
             assert fieldType.docValuesType() == DocValuesType.NONE;
-
-            var field = (MultiValuedBinaryDocValuesField.SeparateCount) context.doc().getByKey(fieldType().name());
-            var countField = (NumericDocValuesField) context.doc().getByKey(fieldType().name() + COUNT_FIELD_SUFFIX);
-            if (field == null) {
-                field = new MultiValuedBinaryDocValuesField.SeparateCount(fieldType().name(), false);
-                context.doc().addWithKey(fieldType().name(), field);
-                countField = NumericDocValuesField.indexedField(field.countFieldName(), -1); // dummy value
-                context.doc().addWithKey(countField.name(), countField);
-            }
-
-            field.add(binaryValue);
-            countField.setLongValue(field.count());
+            MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc(
+                context.doc(),
+                fieldType().name(),
+                binaryValue
+            );
         }
 
         // If we're using binary doc values, then the values are stored in a separate MultiValuedBinaryDocValuesField (see above)
@@ -1483,8 +1488,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
             return SyntheticSourceSupport.FALLBACK;
         }
 
-        boolean docValuesSupportNativeSyntheticSource = fieldType().storedInBinaryDocValues() == false
-            || sourceKeepMode == SourceKeepMode.NONE;
+        boolean docValuesSupportNativeSyntheticSource = fieldType().usesBinaryDocValues() == false || sourceKeepMode == SourceKeepMode.NONE;
 
         if (fieldType.stored() || (docValuesParameters.enabled() && docValuesSupportNativeSyntheticSource)) {
             return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName()));
@@ -1510,7 +1514,7 @@ protected void writeValue(Object value, XContentBuilder b) throws IOException {
                 }
             });
         } else if (docValuesParameters.enabled()) {
-            if (fieldType().storedInBinaryDocValues() == false) {
+            if (fieldType().usesBinaryDocValues() == false) {
                 if (offsetsFieldName != null) {
                     layers.add(new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName));
                 } else {

server/src/main/java/org/elasticsearch/index/mapper/MultiValuedBinaryDocValuesField.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -125,5 +126,19 @@ public BytesRef binaryValue() {
         public String countFieldName() {
             return name() + COUNT_FIELD_SUFFIX;
         }
+
+        public static void addToSeparateCountMultiBinaryFieldInDoc(LuceneDocument doc, String fieldName, BytesRef binaryValue) {
+            var field = (SeparateCount) doc.getByKey(fieldName);
+            var countField = (NumericDocValuesField) doc.getByKey(fieldName + COUNT_FIELD_SUFFIX);
+            if (field == null) {
+                field = new SeparateCount(fieldName, false);
+                countField = NumericDocValuesField.indexedField(field.countFieldName(), -1); // dummy value
+                doc.addWithKey(field.name(), field);
+                doc.addWithKey(countField.name(), countField);
+            }
+
+            field.add(binaryValue);
+            countField.setLongValue(field.count());
+        }
     }
 }