diff --git a/docs/changelog/140246.yaml b/docs/changelog/140246.yaml new file mode 100644 index 0000000000000..a2821e16c378c --- /dev/null +++ b/docs/changelog/140246.yaml @@ -0,0 +1,5 @@ +pr: 140246 +summary: Store flattened field data in binary doc values +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 8233df0610f65..2dc8f3133265a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -217,6 +217,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion MOD_ROUTING_FUNCTION = def(9_064_0_00, Version.LUCENE_10_3_2); public static final IndexVersion FALLBACK_TEXT_FIELDS_BINARY_DOC_VALUES_FORMAT_CHECK = def(9_065_0_00, Version.LUCENE_10_3_2); public static final IndexVersion READ_SI_FILES_FROM_MEMORY_FOR_HOLLOW_COMMITS = def(9_066_0_00, Version.LUCENE_10_3_2); + public static final IndexVersion FLATTENED_FIELD_TSDB_CODEC_USE_BINARY_DOC_VALUES = def(9_067_0_00, Version.LUCENE_10_3_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiValuedSortedBinaryDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiValuedSortedBinaryDocValues.java index cf3116588ba94..327066a067932 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiValuedSortedBinaryDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiValuedSortedBinaryDocValues.java @@ -49,9 +49,15 @@ public static MultiValuedSortedBinaryDocValues from(LeafReader leafReader, Strin if (counts == null) { return new IntegratedCounts(values); } else { + Sparsity sparsity = Sparsity.UNKNOWN; + ValueMode valueMode = ValueMode.UNKNOWN; + DocValuesSkipper countsSkipper = leafReader.getDocValuesSkipper(countsFieldName); - Sparsity sparsity = countsSkipper.docCount() == leafReader.maxDoc() ? Sparsity.DENSE : Sparsity.SPARSE; - ValueMode valueMode = countsSkipper.maxValue() == 1 ? ValueMode.SINGLE_VALUED : ValueMode.MULTI_VALUED; + if (countsSkipper != null) { + sparsity = countsSkipper.docCount() == leafReader.maxDoc() ? Sparsity.DENSE : Sparsity.SPARSE; + valueMode = countsSkipper.maxValue() == 1 ? ValueMode.SINGLE_VALUED : ValueMode.MULTI_VALUED; + } + return new SeparateCounts(values, counts, sparsity, valueMode); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index 33112efde7f38..af27aab92e27b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -202,7 +202,7 @@ public NumericDocValues getNormValues(String field) throws IOException { @Override public DocValuesSkipper getDocValuesSkipper(String s) throws IOException { - throw new UnsupportedOperationException(); + return null; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b6edff142109a..317c1bad5c21c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -16,7 +16,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.InvertableType; -import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesSkipIndexType; @@ -112,7 +111,6 @@ import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; import static org.elasticsearch.index.mapper.FieldMapper.Parameter.useTimeSeriesDocValuesSkippers; -import static org.elasticsearch.index.mapper.MultiValuedBinaryDocValuesField.SeparateCount.COUNT_FIELD_SUFFIX; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -345,6 +343,10 @@ public DocValuesParameter.Values docValuesParameters() { return docValuesParameters.getValue(); } + boolean usesBinaryDocValues() { + return docValuesParameters().enabled() && docValuesParameters().cardinality() == DocValuesParameter.Values.Cardinality.HIGH; + } + public SimilarityProvider similarity() { return this.similarity.get(); } @@ -544,7 +546,7 @@ public static final class KeywordFieldType extends TextFamilyFieldType { private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; - private final DocValuesParameter.Values docValuesParameters; + private final boolean usesBinaryDocValues; public KeywordFieldType( String name, @@ -573,14 +575,24 @@ public KeywordFieldType( this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); - this.docValuesParameters = builder.docValuesParameters(); + this.usesBinaryDocValues = builder.usesBinaryDocValues(); } public KeywordFieldType(String name) { - this(name, true, true, Collections.emptyMap()); + this(name, true, true, false, Collections.emptyMap()); } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { + this(name, isIndexed, hasDocValues, false, meta); + } + + public KeywordFieldType( + String name, + boolean isIndexed, + boolean hasDocValues, + boolean usesBinaryDocValues, + Map meta + ) { super(name, IndexType.terms(isIndexed, hasDocValues), false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta, false, false); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = IGNORE_ABOVE_DEFAULT; @@ -588,7 +600,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.docValuesParameters = DEFAULT_DOC_VALUES_PARAMS; + this.usesBinaryDocValues = usesBinaryDocValues; } public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSource) { @@ -607,7 +619,7 @@ public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSou this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.docValuesParameters = DEFAULT_DOC_VALUES_PARAMS; + this.usesBinaryDocValues = false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -626,7 +638,11 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.docValuesParameters = DEFAULT_DOC_VALUES_PARAMS; + this.usesBinaryDocValues = false; + } + + public boolean usesBinaryDocValues() { + return usesBinaryDocValues; } @Override @@ -639,7 +655,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); if (indexType.hasTerms()) { return super.termQuery(value, context); - } else if (storedInBinaryDocValues()) { + } else if (usesBinaryDocValues) { return new SlowCustomBinaryDocValuesTermQuery(name(), indexedValueForSearch(value)); } else { return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value)); @@ -651,7 +667,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); if (indexType.hasTerms()) { return super.termsQuery(values, context); - } else if (storedInBinaryDocValues()) { + } else if (usesBinaryDocValues) { return new StringScriptFieldTermsQuery( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -675,7 +691,7 @@ public Query rangeQuery( failIfNotIndexedNorDocValuesFallback(context); if (indexType.hasTerms()) { return super.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); - } else if (storedInBinaryDocValues()) { + } else if (usesBinaryDocValues) { return new StringScriptFieldRangeQuery( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -709,7 +725,7 @@ public Query fuzzyQuery( failIfNotIndexedNorDocValuesFallback(context); if (indexType.hasTerms()) { return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context, rewriteMethod); - } else if (storedInBinaryDocValues()) { + } else if (usesBinaryDocValues) { return StringScriptFieldFuzzyQuery.build( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -741,7 +757,7 @@ public Query prefixQuery( failIfNotIndexedNorDocValuesFallback(context); if (indexType.hasTerms()) { return super.prefixQuery(value, method, caseInsensitive, context); - } else if (storedInBinaryDocValues()) { + } else if (usesBinaryDocValues) { return new StringScriptFieldPrefixQuery( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -769,7 +785,7 @@ public Query termQueryCaseInsensitive(Object value, SearchExecutionContext conte failIfNotIndexedNorDocValuesFallback(context); if (indexType.hasTerms()) { return super.termQueryCaseInsensitive(value, context); - } else if (storedInBinaryDocValues()) { + } else if (usesBinaryDocValues) { return new StringScriptFieldTermQuery( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -794,7 +810,7 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi if (indexType.hasTerms()) { terms = MultiTerms.getTerms(reader, name()); } else if (hasDocValues()) { - if (storedInBinaryDocValues()) { + if (usesBinaryDocValues) { throw new UnsupportedOperationException("TODO"); } else { terms = SortedSetDocValuesTerms.getTerms(reader, name()); @@ -828,10 +844,6 @@ public String typeName() { return CONTENT_TYPE; } - public boolean storedInBinaryDocValues() { - return docValuesParameters.enabled() && docValuesParameters.cardinality() == DocValuesParameter.Values.Cardinality.HIGH; - } - @Override public boolean eagerGlobalOrdinals() { return eagerGlobalOrdinals; @@ -846,7 +858,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSourceEnabled())) { BlockLoaderFunctionConfig cfg = blContext.blockLoaderFunctionConfig(); if (cfg == null) { - if (storedInBinaryDocValues()) { + if (usesBinaryDocValues) { return new BytesRefsFromBinaryMultiSeparateCountBlockLoader(name()); } else { return new BytesRefsFromOrdsBlockLoader(name()); @@ -986,7 +998,7 @@ protected BytesRef storedToBytesRef(Object stored) { } private IndexFieldData.Builder fieldDataFromDocValues() { - if (storedInBinaryDocValues()) { + if (usesBinaryDocValues) { return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, KeywordDocValuesField::new); } else { return new SortedSetOrdinalsIndexFieldData.Builder( @@ -1076,7 +1088,7 @@ public Query wildcardQuery( value = indexedValueForSearch(value).utf8ToString(); } - if (storedInBinaryDocValues()) { + if (usesBinaryDocValues) { return new SlowCustomBinaryDocValuesWildcardQuery(name(), value, caseInsensitive); } @@ -1102,7 +1114,7 @@ public Query normalizedWildcardQuery(String value, MultiTermQuery.RewriteMethod value = indexedValueForSearch(value).utf8ToString(); } - if (storedInBinaryDocValues()) { + if (usesBinaryDocValues) { return new StringScriptFieldWildcardQuery( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -1134,7 +1146,7 @@ public Query regexpQuery( throw new IllegalArgumentException("Match flags not yet implemented [" + matchFlags + "]"); } - if (storedInBinaryDocValues()) { + if (usesBinaryDocValues) { return new StringScriptFieldRegexpQuery( new Script(""), ctx -> new SortedBinaryDocValuesStringFieldScript(name(), context.lookup(), ctx), @@ -1365,20 +1377,13 @@ private boolean indexValue(DocumentParserContext context, XContentString value) throw new IllegalArgumentException(msg); } - if (fieldType().storedInBinaryDocValues()) { + if (fieldType().usesBinaryDocValues()) { assert fieldType.docValuesType() == DocValuesType.NONE; - - var field = (MultiValuedBinaryDocValuesField.SeparateCount) context.doc().getByKey(fieldType().name()); - var countField = (NumericDocValuesField) context.doc().getByKey(fieldType().name() + COUNT_FIELD_SUFFIX); - if (field == null) { - field = new MultiValuedBinaryDocValuesField.SeparateCount(fieldType().name(), false); - context.doc().addWithKey(fieldType().name(), field); - countField = NumericDocValuesField.indexedField(field.countFieldName(), -1); // dummy value - context.doc().addWithKey(countField.name(), countField); - } - - field.add(binaryValue); - countField.setLongValue(field.count()); + MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc( + context.doc(), + fieldType().name(), + binaryValue + ); } // If we're using binary doc values, then the values are stored in a separate MultiValuedBinaryDocValuesField (see above) @@ -1483,8 +1488,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { return SyntheticSourceSupport.FALLBACK; } - boolean docValuesSupportNativeSyntheticSource = fieldType().storedInBinaryDocValues() == false - || sourceKeepMode == SourceKeepMode.NONE; + boolean docValuesSupportNativeSyntheticSource = fieldType().usesBinaryDocValues() == false || sourceKeepMode == SourceKeepMode.NONE; if (fieldType.stored() || (docValuesParameters.enabled() && docValuesSupportNativeSyntheticSource)) { return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName())); @@ -1510,7 +1514,7 @@ protected void writeValue(Object value, XContentBuilder b) throws IOException { } }); } else if (docValuesParameters.enabled()) { - if (fieldType().storedInBinaryDocValues() == false) { + if (fieldType().usesBinaryDocValues() == false) { if (offsetsFieldName != null) { layers.add(new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName)); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MultiValuedBinaryDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/MultiValuedBinaryDocValuesField.java index d036b74fe94ba..d976608370c5d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MultiValuedBinaryDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MultiValuedBinaryDocValuesField.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -125,5 +126,19 @@ public BytesRef binaryValue() { public String countFieldName() { return name() + COUNT_FIELD_SUFFIX; } + + public static void addToSeparateCountMultiBinaryFieldInDoc(LuceneDocument doc, String fieldName, BytesRef binaryValue) { + var field = (SeparateCount) doc.getByKey(fieldName); + var countField = (NumericDocValuesField) doc.getByKey(fieldName + COUNT_FIELD_SUFFIX); + if (field == null) { + field = new SeparateCount(fieldName, false); + countField = NumericDocValuesField.indexedField(field.countFieldName(), -1); // dummy value + doc.addWithKey(field.name(), field); + doc.addWithKey(countField.name(), countField); + } + + field.add(binaryValue); + countField.setLongValue(field.count()); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/BinaryKeyedFlattenedLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/BinaryKeyedFlattenedLeafFieldData.java new file mode 100644 index 0000000000000..ad653275121e3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/BinaryKeyedFlattenedLeafFieldData.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.flattened; + +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; + +import java.io.IOException; +import java.util.Collection; + +/** + * The atomic field data implementation for {@link FlattenedFieldMapper.KeyedFlattenedFieldType}. + * + * This class wraps the field data that is built directly on the keyed flattened field, + * and filters out values whose prefix doesn't match the requested key. + */ +public final class BinaryKeyedFlattenedLeafFieldData implements LeafFieldData { + + private final String key; + private final LeafFieldData delegate; + private final ToScriptFieldFactory toScriptFieldFactory; + + private static final SortedBinaryDocValues EMPTY = new SortedBinaryDocValues() { + @Override + public boolean advanceExact(int doc) throws IOException { + return false; + } + + @Override + public int docValueCount() { + return 0; + } + + @Override + public BytesRef nextValue() throws IOException { + return null; + } + }; + + BinaryKeyedFlattenedLeafFieldData( + String key, + LeafFieldData delegate, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.key = key; + this.delegate = delegate; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + @Override + public long ramBytesUsed() { + return delegate.ramBytesUsed(); + } + + @Override + public Collection getChildResources() { + return delegate.getChildResources(); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory(getBytesValues(), name); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new KeyedFlattenedBinaryDocValues(new BytesRef(key), delegate.getBytesValues()); + } + + private static int compare(BytesRef key, BytesRef term) { + BytesRef extractedKey = FlattenedFieldParser.extractKey(term); + return key.compareTo(extractedKey); + } + + private static class KeyedFlattenedBinaryDocValues extends SortedBinaryDocValues { + + private final BytesRef key; + private final SortedBinaryDocValues delegate; + private int count; + private int seen; + + private KeyedFlattenedBinaryDocValues(BytesRef key, SortedBinaryDocValues delegate) { + this.key = key; + this.delegate = delegate; + } + + @Override + public int docValueCount() { + return count; + } + + @Override + public boolean advanceExact(int target) throws IOException { + this.seen = 0; + + if (delegate.advanceExact(target)) { + int offset = -1; + int count = 0; + for (int i = 0; i < delegate.docValueCount(); i++) { + BytesRef value = delegate.nextValue(); + int comparison = compare(key, value); + if (comparison == 0) { + if (offset < 0) { + offset = i; + } + count++; + } else if (comparison < 0) { + break; + } + } + this.count = count; + if (count == 0) { + return false; + } + + // It is a match, but still need to reset the iterator on the current doc and + // iterate the delegate until at least offset has been seen. + boolean advanced = delegate.advanceExact(target); + assert advanced; + + for (int i = 0; i < offset; ++i) { + delegate.nextValue(); + } + return true; + } + + this.count = 0; + return false; + } + + @Override + public BytesRef nextValue() throws IOException { + if (seen >= count) { + return null; + } + seen++; + BytesRef keyedValue = delegate.nextValue(); + int prefixLength = key.length + 1; + int valueLength = keyedValue.length - prefixLength; + return new BytesRef(keyedValue.bytes, keyedValue.offset + prefixLength, valueLength); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 547f0def112f6..df07a3dfcb704 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.ImpactsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.OrdinalMap; @@ -41,6 +40,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -48,8 +48,11 @@ import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.plain.BytesBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DynamicFieldType; @@ -184,17 +187,24 @@ public static class Builder extends FieldMapper.Builder { private final IndexMode indexMode; private final IndexVersion indexCreatedVersion; + private final boolean usesBinaryDocValues; public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } + private static boolean usesBinaryDocValues(IndexSettings indexSettings) { + return indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.FLATTENED_FIELD_TSDB_CODEC_USE_BINARY_DOC_VALUES) + && indexSettings.useTimeSeriesDocValuesFormat(); + } + public Builder(final String name) { this( name, IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()), IndexMode.STANDARD, - IndexVersion.current() + IndexVersion.current(), + false ); } @@ -203,17 +213,25 @@ private Builder(String name, MappingParserContext mappingParserContext) { name, IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), mappingParserContext.getIndexSettings().getMode(), - mappingParserContext.indexVersionCreated() + mappingParserContext.indexVersionCreated(), + usesBinaryDocValues(mappingParserContext.getIndexSettings()) ); } - private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { + private Builder( + String name, + int ignoreAboveDefault, + IndexMode indexMode, + IndexVersion indexCreatedVersion, + boolean usesBinaryDocValues + ) { super(name); this.ignoreAboveDefault = ignoreAboveDefault; this.indexMode = indexMode; this.indexCreatedVersion = indexCreatedVersion; this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); this.dimensions.precludesParameters(ignoreAbove); + this.usesBinaryDocValues = usesBinaryDocValues; } @Override @@ -248,7 +266,8 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), dimensions.get(), - new IgnoreAbove(ignoreAbove.getValue(), indexMode, indexCreatedVersion) + new IgnoreAbove(ignoreAbove.getValue(), indexMode, indexCreatedVersion), + usesBinaryDocValues ); return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } @@ -264,6 +283,7 @@ public static final class KeyedFlattenedFieldType extends StringFieldType { private final String key; private final String rootName; private final boolean isDimension; + private final boolean usesBinaryDocValues; @Override public boolean isDimension() { @@ -276,7 +296,8 @@ public boolean isDimension() { String key, boolean splitQueriesOnWhitespace, Map meta, - boolean isDimension + boolean isDimension, + boolean usesBinaryDocValues ) { super( rootName + KEYED_FIELD_SUFFIX, @@ -288,10 +309,19 @@ public boolean isDimension() { this.key = key; this.rootName = rootName; this.isDimension = isDimension; + this.usesBinaryDocValues = usesBinaryDocValues; } - private KeyedFlattenedFieldType(String rootName, String key, RootFlattenedFieldType ref) { - this(rootName, ref.indexType(), key, ref.splitQueriesOnWhitespace, ref.meta(), ref.dimensions.contains(key)); + private KeyedFlattenedFieldType(String rootName, String key, RootFlattenedFieldType ref, boolean usesBinaryDocValues) { + this( + rootName, + ref.indexType(), + key, + ref.splitQueriesOnWhitespace, + ref.meta(), + ref.dimensions.contains(key), + usesBinaryDocValues + ); } @Override @@ -421,7 +451,12 @@ public BytesRef indexedValueForSearch(Object value) { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { failIfNoDocValues(); - return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)); + + if (usesBinaryDocValues) { + return new BinaryKeyedFlattenedFieldData.Builder(name(), key, FlattenedDocValuesField::new); + } else { + return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)); + } } @Override @@ -661,6 +696,86 @@ public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService } } + public static final class BinaryKeyedFlattenedFieldData implements IndexFieldData { + private final String key; + private final BytesBinaryIndexFieldData delegate; + private final ToScriptFieldFactory toScriptFieldFactory; + + private BinaryKeyedFlattenedFieldData( + String key, + BytesBinaryIndexFieldData delegate, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.delegate = delegate; + this.key = key; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + public String getKey() { + return key; + } + + @Override + public String getFieldName() { + return delegate.getFieldName(); + } + + @Override + public ValuesSourceType getValuesSourceType() { + return delegate.getValuesSourceType(); + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); + return new SortField(getFieldName(), source, reverse); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("only supported on numeric fields"); + } + + @Override + public LeafFieldData load(LeafReaderContext context) { + LeafFieldData fieldData = delegate.load(context); + return new BinaryKeyedFlattenedLeafFieldData(key, fieldData, toScriptFieldFactory); + } + + @Override + public LeafFieldData loadDirect(LeafReaderContext context) throws Exception { + LeafFieldData fieldData = delegate.loadDirect(context); + return new BinaryKeyedFlattenedLeafFieldData(key, fieldData, toScriptFieldFactory); + } + + public static class Builder implements IndexFieldData.Builder { + private final String fieldName; + private final String key; + private final ToScriptFieldFactory toScriptFieldFactory; + + Builder(String fieldName, String key, ToScriptFieldFactory toScriptFieldFactory) { + this.fieldName = fieldName; + this.key = key; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + @Override + public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + var delegate = new BytesBinaryIndexFieldData(fieldName, CoreValuesSourceType.KEYWORD, toScriptFieldFactory); + return new BinaryKeyedFlattenedFieldData(key, delegate, toScriptFieldFactory); + } + } + } + /** * A field type that represents all 'root' values. This field type is used in * searches on the flattened field itself, e.g. 'my_flattened: some_value'. @@ -671,6 +786,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme private final List dimensions; private final boolean isDimension; private final IgnoreAbove ignoreAbove; + private final boolean usesBinaryDocValues; RootFlattenedFieldType( String name, @@ -678,9 +794,19 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, - IgnoreAbove ignoreAbove + IgnoreAbove ignoreAbove, + boolean usesBinaryDocValues ) { - this(name, indexType, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); + this( + name, + indexType, + meta, + splitQueriesOnWhitespace, + eagerGlobalOrdinals, + Collections.emptyList(), + ignoreAbove, + usesBinaryDocValues + ); } RootFlattenedFieldType( @@ -690,7 +816,8 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, List dimensions, - IgnoreAbove ignoreAbove + IgnoreAbove ignoreAbove, + boolean usesBinaryDocValues ) { super( name, @@ -704,6 +831,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme this.dimensions = dimensions; this.isDimension = dimensions.isEmpty() == false; this.ignoreAbove = ignoreAbove; + this.usesBinaryDocValues = usesBinaryDocValues; } @Override @@ -728,11 +856,16 @@ public Object valueForDisplay(Object value) { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { failIfNoDocValues(); - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n) - ); + + if (usesBinaryDocValues) { + return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, FlattenedDocValuesField::new); + } else { + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n) + ); + } } @Override @@ -809,11 +942,17 @@ private Object filterIgnoredValues(final Object entryValue) { @Override public MappedFieldType getChildFieldType(String childPath) { - return new KeyedFlattenedFieldType(name(), childPath, this); + return new KeyedFlattenedFieldType(name(), childPath, this, usesBinaryDocValues); } public MappedFieldType getKeyedFieldType() { - return new KeywordFieldMapper.KeywordFieldType(name() + KEYED_FIELD_SUFFIX); + return new KeywordFieldMapper.KeywordFieldType( + name() + KEYED_FIELD_SUFFIX, + true, + true, + usesBinaryDocValues, + Collections.emptyMap() + ); } @Override @@ -847,7 +986,8 @@ private FlattenedFieldMapper(String leafName, MappedFieldType mappedFieldType, B mappedFieldType, builder.depthLimit.get(), builder.ignoreAbove.get(), - builder.nullValue.get() + builder.nullValue.get(), + builder.usesBinaryDocValues ); } @@ -889,8 +1029,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio try { // make sure that we don't expand dots in field names while parsing context.path().setWithinLeafObject(true); - List fields = fieldParser.parse(context); - context.doc().addAll(fields); + fieldParser.parse(context); } finally { context.path().setWithinLeafObject(false); } @@ -902,7 +1041,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), builder.ignoreAboveDefault, builder.indexMode, builder.indexCreatedVersion).init(this); + return new Builder( + leafName(), + builder.ignoreAboveDefault, + builder.indexMode, + builder.indexCreatedVersion, + builder.usesBinaryDocValues + ).init(this); } @Override @@ -913,7 +1058,8 @@ protected SyntheticSourceSupport syntheticSourceSupport() { fullPath(), fullPath() + KEYED_FIELD_SUFFIX, fieldType().ignoreAbove.valuesPotentiallyIgnored() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, - leafName() + leafName(), + builder.usesBinaryDocValues ) ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 529e5aac3ea16..1ac9f75aa4a59 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -14,17 +14,15 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MultiValuedBinaryDocValuesField; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; /** * A helper class for {@link FlattenedFieldMapper} parses a JSON object @@ -43,6 +41,8 @@ class FlattenedFieldParser { private final int ignoreAbove; private final String nullValue; + private final boolean usesBinaryDocValues; + FlattenedFieldParser( String rootFieldFullPath, String keyedFieldFullPath, @@ -50,7 +50,8 @@ class FlattenedFieldParser { MappedFieldType fieldType, int depthLimit, int ignoreAbove, - String nullValue + String nullValue, + boolean usesBinaryDocValues ) { this.rootFieldFullPath = rootFieldFullPath; this.keyedFieldFullPath = keyedFieldFullPath; @@ -59,22 +60,20 @@ class FlattenedFieldParser { this.depthLimit = depthLimit; this.ignoreAbove = ignoreAbove; this.nullValue = nullValue; + this.usesBinaryDocValues = usesBinaryDocValues; } - public List parse(final DocumentParserContext documentParserContext) throws IOException { + public void parse(final DocumentParserContext documentParserContext) throws IOException { XContentParser parser = documentParserContext.parser(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); ContentPath path = new ContentPath(); - List fields = new ArrayList<>(); var context = new Context(parser, documentParserContext); - parseObject(context, path, fields); - - return fields; + parseObject(context, path); } - private void parseObject(Context context, ContentPath path, List fields) throws IOException { + private void parseObject(Context context, ContentPath path) throws IOException { String currentName = null; XContentParser parser = context.parser(); while (true) { @@ -87,43 +86,37 @@ private void parseObject(Context context, ContentPath path, List currentName = parser.currentName(); } else { assert currentName != null; - parseFieldValue(context, token, path, currentName, fields); + parseFieldValue(context, token, path, currentName); } } } - private void parseArray(Context context, ContentPath path, String currentName, List fields) throws IOException { + private void parseArray(Context context, ContentPath path, String currentName) throws IOException { XContentParser parser = context.parser(); while (true) { XContentParser.Token token = parser.nextToken(); if (token == XContentParser.Token.END_ARRAY) { return; } - parseFieldValue(context, token, path, currentName, fields); + parseFieldValue(context, token, path, currentName); } } - private void parseFieldValue( - Context context, - XContentParser.Token token, - ContentPath path, - String currentName, - List fields - ) throws IOException { + private void parseFieldValue(Context context, XContentParser.Token token, ContentPath path, String currentName) throws IOException { XContentParser parser = context.parser(); if (token == XContentParser.Token.START_OBJECT) { path.add(currentName); validateDepthLimit(path); - parseObject(context, path, fields); + parseObject(context, path); path.remove(); } else if (token == XContentParser.Token.START_ARRAY) { - parseArray(context, path, currentName, fields); + parseArray(context, path, currentName); } else if (token.isValue()) { String value = parser.text(); - addField(context, path, currentName, value, fields); + addField(context, path, currentName, value); } else if (token == XContentParser.Token.VALUE_NULL) { if (nullValue != null) { - addField(context, path, currentName, nullValue, fields); + addField(context, path, currentName, nullValue); } } else { // Note that we throw an exception here just to be safe. We don't actually expect to reach @@ -132,7 +125,7 @@ private void parseFieldValue( } } - private void addField(Context context, ContentPath path, String currentName, String value, List fields) { + private void addField(Context context, ContentPath path, String currentName, String value) { String key = path.pathAsText(currentName); if (key.contains(SEPARATOR)) { throw new IllegalArgumentException( @@ -145,7 +138,7 @@ private void addField(Context context, ContentPath path, String currentName, Str if (value.length() > ignoreAbove) { if (context.documentParserContext().mappingLookup().isSourceSynthetic()) { - fields.add(new StoredField(keyedIgnoredValuesFieldFullPath, bytesKeyedValue)); + context.documentParserContext.doc().add(new StoredField(keyedIgnoredValuesFieldFullPath, bytesKeyedValue)); } return; } @@ -169,13 +162,26 @@ private void addField(Context context, ContentPath path, String currentName, Str } BytesRef bytesValue = new BytesRef(value); if (fieldType.indexType().hasTerms()) { - fields.add(new StringField(rootFieldFullPath, bytesValue, Field.Store.NO)); - fields.add(new StringField(keyedFieldFullPath, bytesKeyedValue, Field.Store.NO)); + context.documentParserContext.doc().add(new StringField(rootFieldFullPath, bytesValue, Field.Store.NO)); + context.documentParserContext.doc().add(new StringField(keyedFieldFullPath, bytesKeyedValue, Field.Store.NO)); } if (fieldType.hasDocValues()) { - fields.add(new SortedSetDocValuesField(rootFieldFullPath, bytesValue)); - fields.add(new SortedSetDocValuesField(keyedFieldFullPath, bytesKeyedValue)); + if (usesBinaryDocValues) { + MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc( + context.documentParserContext.doc(), + rootFieldFullPath, + bytesValue + ); + MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc( + context.documentParserContext.doc(), + keyedFieldFullPath, + bytesKeyedValue + ); + } else { + context.documentParserContext.doc().add(new SortedSetDocValuesField(rootFieldFullPath, bytesValue)); + context.documentParserContext.doc().add(new SortedSetDocValuesField(keyedFieldFullPath, bytesKeyedValue)); + } if (fieldType.isDimension() == false || context.documentParserContext().getRoutingFields().isNoop()) { return; @@ -219,7 +225,7 @@ static BytesRef extractValue(BytesRef keyedValue) { } } int valueStart = keyedValue.offset + length + 1; - return new BytesRef(keyedValue.bytes, valueStart, keyedValue.length - valueStart); + return new BytesRef(keyedValue.bytes, valueStart, keyedValue.length - (length + 1)); } private record Context(XContentParser parser, DocumentParserContext documentParserContext) {} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java index 5493bcddb4a84..e6e3db1f175ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java @@ -164,6 +164,7 @@ public boolean equals(Object obj) { } } + @FunctionalInterface public interface SortedKeyedValues { BytesRef next() throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java index f957d7ce01902..16855e939929e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java @@ -14,6 +14,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.fielddata.MultiValuedSortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.xcontent.XContentBuilder; @@ -29,6 +31,7 @@ class FlattenedSortedSetDocValuesSyntheticFieldLoader implements SourceLoader.Sy private final String keyedFieldFullPath; private final String keyedIgnoredValuesFieldFullPath; private final String leafName; + private final boolean usesBinaryDocValues; private DocValuesFieldValues docValues = NO_VALUES; private List ignoredValues = List.of(); @@ -46,12 +49,14 @@ class FlattenedSortedSetDocValuesSyntheticFieldLoader implements SourceLoader.Sy String fieldFullPath, String keyedFieldFullPath, @Nullable String keyedIgnoredValuesFieldFullPath, - String leafName + String leafName, + boolean usesBinaryDocValues ) { this.fieldFullPath = fieldFullPath; this.keyedFieldFullPath = keyedFieldFullPath; this.keyedIgnoredValuesFieldFullPath = keyedIgnoredValuesFieldFullPath; this.leafName = leafName; + this.usesBinaryDocValues = usesBinaryDocValues; } @Override @@ -73,14 +78,27 @@ public Stream> storedFieldLoaders() { @Override public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException { - final SortedSetDocValues dv = DocValues.getSortedSet(reader, keyedFieldFullPath); - if (dv.getValueCount() == 0) { - docValues = NO_VALUES; - return null; - } - final FlattenedFieldDocValuesLoader loader = new FlattenedFieldDocValuesLoader(dv); - docValues = loader; - return loader; + if (usesBinaryDocValues) { + var binaryDv = reader.getBinaryDocValues(keyedFieldFullPath); + if (binaryDv == null) { + return null; + } + + SortedBinaryDocValues dv = MultiValuedSortedBinaryDocValues.from(reader, keyedFieldFullPath, binaryDv); + MultiValuedBinaryFieldValues loader = new MultiValuedBinaryFieldValues(dv); + docValues = loader; + return loader; + } else { + final SortedSetDocValues dv = DocValues.getSortedSet(reader, keyedFieldFullPath); + if (dv.getValueCount() == 0) { + docValues = NO_VALUES; + return null; + } + + SortedSetFieldValues loader = new SortedSetFieldValues(dv); + docValues = loader; + return loader; + } } @Override @@ -94,7 +112,7 @@ public void write(XContentBuilder b) throws IOException { return; } - FlattenedFieldSyntheticWriterHelper.SortedKeyedValues sortedKeyedValues = new DocValuesSortedKeyedValues(docValues); + FlattenedFieldSyntheticWriterHelper.SortedKeyedValues sortedKeyedValues = docValues.getValues(); if (ignoredValues.isEmpty() == false) { var ignoredValuesSet = new TreeSet(); for (Object value : ignoredValues) { @@ -118,7 +136,7 @@ public void reset() { private interface DocValuesFieldValues { int count(); - SortedSetDocValues getValues(); + FlattenedFieldSyntheticWriterHelper.SortedKeyedValues getValues(); } private static final DocValuesFieldValues NO_VALUES = new DocValuesFieldValues() { @@ -128,36 +146,78 @@ public int count() { } @Override - public SortedSetDocValues getValues() { - return null; + public FlattenedFieldSyntheticWriterHelper.SortedKeyedValues getValues() { + return () -> null; } }; - /** - * Load ordinals in line with populating the doc and immediately - * convert from ordinals into {@link BytesRef}s. - */ - private static class FlattenedFieldDocValuesLoader implements DocValuesLoader, DocValuesFieldValues { - private final SortedSetDocValues dv; + private static final class SortedSetFieldValues implements DocValuesFieldValues, DocValuesLoader { + private final SortedSetDocValues docValues; private boolean hasValue; - FlattenedFieldDocValuesLoader(final SortedSetDocValues dv) { - this.dv = dv; + SortedSetFieldValues(SortedSetDocValues docValues) { + this.docValues = docValues; + } + + @Override + public boolean advanceToDoc(int docId) throws IOException { + return hasValue = docValues.advanceExact(docId); + } + + @Override + public int count() { + return hasValue ? docValues.docValueCount() : 0; + } + + @Override + public FlattenedFieldSyntheticWriterHelper.SortedKeyedValues getValues() { + return new FlattenedFieldSyntheticWriterHelper.SortedKeyedValues() { + private int seen = 0; + + @Override + public BytesRef next() throws IOException { + if (seen < count()) { + seen += 1; + return docValues.lookupOrd(docValues.nextOrd()); + } + return null; + } + }; + } + } + + private static final class MultiValuedBinaryFieldValues implements DocValuesFieldValues, DocValuesLoader { + private final SortedBinaryDocValues docValues; + private boolean hasValue = false; + + MultiValuedBinaryFieldValues(SortedBinaryDocValues docValues) { + this.docValues = docValues; } @Override public boolean advanceToDoc(int docId) throws IOException { - return hasValue = dv.advanceExact(docId); + return hasValue = docValues.advanceExact(docId); } @Override public int count() { - return hasValue ? dv.docValueCount() : 0; + return hasValue ? docValues.docValueCount() : 0; } @Override - public SortedSetDocValues getValues() { - return dv; + public FlattenedFieldSyntheticWriterHelper.SortedKeyedValues getValues() { + return new FlattenedFieldSyntheticWriterHelper.SortedKeyedValues() { + private int seen = 0; + + @Override + public BytesRef next() throws IOException { + if (seen < count()) { + seen += 1; + return docValues.nextValue(); + } + return null; + } + }; } } @@ -207,23 +267,4 @@ public BytesRef next() throws IOException { } } - private static class DocValuesSortedKeyedValues implements FlattenedFieldSyntheticWriterHelper.SortedKeyedValues { - private final DocValuesFieldValues docValues; - private int seen = 0; - - private DocValuesSortedKeyedValues(DocValuesFieldValues docValues) { - this.docValues = docValues; - } - - @Override - public BytesRef next() throws IOException { - if (seen < docValues.count()) { - seen += 1; - var sortedSetDocValues = docValues.getValues(); - return sortedSetDocValues.lookupOrd(sortedSetDocValues.nextOrd()); - } - - return null; - } - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 6eec91ce826b5..84901fd5e751b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -144,6 +144,29 @@ public void testDefaults() throws Exception { assertEquals(0, fieldNamesFields.size()); } + public void testBinaryDocValuesType() throws Exception { + DocumentMapper mapper = createMapperService( + Settings.builder().put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), true).build(), + fieldMapping(this::minimalMapping) + ).documentMapper(); + ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject())); + + List fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(2, fields.size()); + + // Check the keyed fields. + List keyedFields = parsedDoc.rootDoc().getFields("field._keyed"); + assertEquals(2, keyedFields.size()); + + assertEquals("field", fields.get(1).name()); + assertEquals(new BytesRef("value"), fields.get(1).binaryValue()); + assertEquals(DocValuesType.BINARY, fields.get(1).fieldType().docValuesType()); + + assertEquals("field._keyed", keyedFields.get(1).name()); + assertEquals(new BytesRef("key\0value"), keyedFields.get(1).binaryValue()); + assertEquals(DocValuesType.BINARY, keyedFields.get(1).fieldType().docValuesType()); + } + public void testNotDimension() throws Exception { MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping)); FlattenedFieldMapper.RootFlattenedFieldType ft = (FlattenedFieldMapper.RootFlattenedFieldType) mapperService.fieldType("field"); @@ -247,6 +270,25 @@ public void testDisableIndex() throws Exception { assertEquals(DocValuesType.SORTED_SET, keyedFields.get(0).fieldType().docValuesType()); } + public void testDisableIndexBinary() throws Exception { + DocumentMapper mapper = createMapperService( + Settings.builder().put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), true).build(), + fieldMapping(b -> { + b.field("type", "flattened"); + b.field("index", false); + }) + ).documentMapper(); + ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject())); + + List fields = parsedDoc.rootDoc().getFields("field"); + assertEquals(1, fields.size()); + assertEquals(DocValuesType.BINARY, fields.get(0).fieldType().docValuesType()); + + List keyedFields = parsedDoc.rootDoc().getFields("field._keyed"); + assertEquals(1, keyedFields.size()); + assertEquals(DocValuesType.BINARY, keyedFields.get(0).fieldType().docValuesType()); + } + public void testDisableDocValues() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java index 68be241ca1885..a60b9e4252ae0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java @@ -40,7 +40,8 @@ public void setUp() throws Exception { new FakeFieldType("field"), Integer.MAX_VALUE, Integer.MAX_VALUE, - null + null, + false ); } @@ -50,7 +51,8 @@ public void testTextValues() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(4, fields.size()); IndexableField field1 = fields.get(0); @@ -75,7 +77,8 @@ public void testNumericValues() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(2, fields.size()); IndexableField field = fields.get(0); @@ -92,7 +95,8 @@ public void testBooleanValues() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(2, fields.size()); IndexableField field = fields.get(0); @@ -110,7 +114,8 @@ public void testBasicArrays() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(4, fields.size()); IndexableField field1 = fields.get(0); @@ -136,7 +141,8 @@ public void testArrayOfArrays() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(6, fields.size()); IndexableField field1 = fields.get(0); @@ -173,7 +179,8 @@ public void testArraysOfObjects() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(6, fields.size()); IndexableField field1 = fields.get(0); @@ -214,7 +221,8 @@ public void testNestedObjects() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(4, fields.size()); IndexableField field1 = fields.get(0); @@ -251,7 +259,8 @@ public void testDottedPaths() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(6, fields.size()); IndexableField field1 = fields.get(0); @@ -295,7 +304,8 @@ public void testDepthLimit() throws Exception { new FakeFieldType("field"), 2, Integer.MAX_VALUE, - null + null, + false ); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); @@ -319,11 +329,13 @@ public void testDepthLimitBoundary() throws Exception { new FakeFieldType("field"), 3, Integer.MAX_VALUE, - null + null, + false ); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = configuredParser.parse(context); + configuredParser.parse(context); + List fields = context.doc().getFields(); assertEquals(4, fields.size()); } @@ -337,11 +349,13 @@ public void testIgnoreAbove() throws Exception { new FakeFieldType("field"), Integer.MAX_VALUE, 10, - null + null, + false ); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = configuredParser.parse(context); + configuredParser.parse(context); + List fields = context.doc().getFields(); assertEquals(0, fields.size()); } @@ -349,7 +363,8 @@ public void testNullValues() throws Exception { String input = "{ \"key\": null}"; TestDocumentParserContext fieldsContext = new TestDocumentParserContext(createXContentParser(input)); - List fields = parser.parse(fieldsContext); + parser.parse(fieldsContext); + List fields = fieldsContext.doc().getFields(); assertEquals(0, fields.size()); MappedFieldType fieldType = new FakeFieldType("field"); @@ -360,11 +375,13 @@ public void testNullValues() throws Exception { fieldType, Integer.MAX_VALUE, Integer.MAX_VALUE, - "placeholder" + "placeholder", + false ); TestDocumentParserContext configuredContext = new TestDocumentParserContext(createXContentParser(input)); - fields = configuredParser.parse(configuredContext); + configuredParser.parse(configuredContext); + fields = configuredContext.doc().getFields(); assertEquals(2, fields.size()); IndexableField field = fields.get(0); @@ -389,7 +406,8 @@ public void testEmptyObject() throws Exception { XContentParser xContentParser = createXContentParser(input); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertEquals(0, fields.size()); } @@ -410,7 +428,8 @@ public void testRandomFields() throws Exception { XContentParser xContentParser = createXContentParser(input.utf8ToString()); TestDocumentParserContext context = new TestDocumentParserContext(xContentParser); - List fields = parser.parse(context); + parser.parse(context); + List fields = context.doc().getFields(); assertTrue(fields.size() > 4); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index 6e1d8143bb24d..51157047b161a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -43,7 +43,7 @@ public class KeyedFlattenedFieldTypeTests extends FieldTypeTestCase { private static KeyedFlattenedFieldType createFieldType() { - return new KeyedFlattenedFieldType("field", IndexType.terms(true, true), "key", false, Collections.emptyMap(), false); + return new KeyedFlattenedFieldType("field", IndexType.terms(true, true), "key", false, Collections.emptyMap(), false, true); } public void testIndexedValueForSearch() { @@ -74,7 +74,8 @@ public void testTermQuery() { "key", false, Collections.emptyMap(), - false + false, + true ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [" + ft.name() + "] since it is not indexed.", e.getMessage()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index eda9a793e73c3..20e528d744509 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -45,7 +45,8 @@ private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { Collections.emptyMap(), false, false, - new Mapper.IgnoreAbove(ignoreAbove) + new Mapper.IgnoreAbove(ignoreAbove), + true ); } @@ -72,7 +73,8 @@ public void testTermQuery() { Collections.emptyMap(), false, false, - IGNORE_ABOVE + IGNORE_ABOVE, + true ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -85,7 +87,8 @@ public void testExistsQuery() { Collections.emptyMap(), false, false, - IGNORE_ABOVE + IGNORE_ABOVE, + true ); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); @@ -95,7 +98,8 @@ public void testExistsQuery() { Collections.emptyMap(), false, false, - IGNORE_ABOVE + IGNORE_ABOVE, + true ); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index 292c47febda43..5d643cdb5afc8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -11,6 +11,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericLongValues; import org.elasticsearch.index.mapper.MapperService; @@ -442,4 +444,24 @@ public void testFlattened() throws Exception { assertFalse(valuesSourceWithOrdinals.supportsGlobalOrdinalsMapping()); }); } + + public void testFlattenedBinary() throws Exception { + MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), true).build(), + fieldMapping(b -> b.field("type", "flattened")) + ); + withAggregationContext(mapperService, List.of(source(b -> b.startObject("field").field("key", "abc").endObject())), context -> { + ValuesSourceConfig config; + config = ValuesSourceConfig.resolve(context, null, "field.key", null, null, null, null, CoreValuesSourceType.KEYWORD); + + ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.getValuesSource(); + LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0); + SortedBinaryDocValues values = valuesSource.bytesValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); + + assertFalse(config.hasOrdinals()); + }); + } }