diff --git a/CHANGELOG.md b/CHANGELOG.md index fb1eb408bbf9f..7b0277386c1c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/)) - Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988)) - Add pull-based ingestion error metrics and make internal queue size configurable ([#18088](https://github.com/opensearch-project/OpenSearch/pull/18088)) +- Adding support for derive source feature and implementing it for various type of field mappers ([#17759](https://github.com/opensearch-project/OpenSearch/pull/17759)) - [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989)) - Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139)) - Allow to get the search request from the QueryCoordinatorContext ([#17818](https://github.com/opensearch-project/OpenSearch/pull/17818)) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index b46b58f415cfd..6a306977f840d 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -500,6 +500,29 @@ private static double objectToDouble(Object value) { return doubleValue; } + @Override + protected void canDeriveSourceInternal() { + checkStoredAndDocValuesForDerivedSource(); + } + + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + *

+ * Considerations: + * 1. When using doc values, for multi value field, result would be in sorted order + * 2. There might be precision loss as values are stored as long after multiplying it with "scaling_factor" for + * both doc values and stored field + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator( + mappedFieldType, + new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()), + new StoredFieldFetcher(mappedFieldType, simpleName()) + ); + } + private static class ScaledFloatIndexFieldData extends IndexNumericFieldData { private final IndexNumericFieldData scaledFieldData; diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java index 7dcfe1848511e..1d656612473c7 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -32,8 +32,15 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.Directory; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; @@ -54,6 +61,8 @@ public class ScaledFloatFieldMapperTests extends MapperTestCase { + private static final String FIELD_NAME = "field"; + @Override protected Collection getPlugins() { return singletonList(new MapperExtrasModulePlugin()); @@ -383,6 +392,78 @@ public void testNullValue() throws IOException { assertFalse(dvField.fieldType().stored()); } + public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException { + ScaledFloatFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + ScaledFloatFieldMapper mapper = getMapper(copyTo, true, true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + ScaledFloatFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), true, false); + float value = 11.523f; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(value, true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + 11.52 + "}", source); + } + } + } + + public void testDerivedValueFetching_StoredField() throws IOException { + try (Directory directory = newDirectory()) { + ScaledFloatFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, true); + float value = 11.523f; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(value, false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + 11.52 + "}", source); + } + } + } + + private ScaledFloatFieldMapper getMapper(FieldMapper.CopyTo copyTo, boolean hasDocValues, boolean isStored) throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "scaled_float").field("store", isStored).field("doc_values", hasDocValues).field("scaling_factor", 100) + ) + ); + ScaledFloatFieldMapper mapper = (ScaledFloatFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private Document createDocument(double value, boolean hasDocValues) { + long scaledValue = Math.round(value * 100); + Document doc = new Document(); + if (hasDocValues) { + doc.add(new SortedNumericDocValuesField(FIELD_NAME, scaledValue)); + } else { + doc.add(new StoredField(FIELD_NAME, scaledValue)); + } + return doc; + } + /** * `index_options` was deprecated and is rejected as of 7.0 */ diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index b4cf585c1329d..ea4cff42ca905 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -412,4 +412,34 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected void canDeriveSourceInternal() { + checkStoredAndDocValuesForDerivedSource(); + } + + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + * + *

+ * Considerations: + * 1. Result will be in boolean type and not in the provided string value type at time of ingestion, + * i.e. [false, "false", ""] will become boolean false + * 2. When using doc values, for multi value field, result will be in sorted order, i.e. at start there will + * be 0 or more false and at end there will be 0 or more true + * 2. When using stored field, for multi value field order would be preserved + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) { + @Override + public Object convert(Object value) { + Long val = (Long) value; + if (val == null) { + return null; + } + return val == 1; + } + }, new StoredFieldFetcher(mappedFieldType, simpleName())); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java index 14b54b803b62a..84557563d8cd3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java @@ -8,6 +8,7 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; @@ -25,6 +26,7 @@ import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.regex.Regex; import org.opensearch.common.time.DateMathParser; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.ConstantIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -74,6 +76,26 @@ private static ConstantKeywordFieldMapper toType(FieldMapper in) { return (ConstantKeywordFieldMapper) in; } + @Override + public void canDeriveSource() { + if (this.copyTo() != null && !this.copyTo().copyToFields().isEmpty()) { + throw new UnsupportedOperationException("Unable to derive source for fields with copy_to parameter set"); + } + } + + /** + * For each doc, it will return constant value defined in field mapping + *

+ * Note: Doc for which, field in absent, deriveSource will still consider the still to be present, and it will + * return the same. + */ + @Override + public void deriveSource(XContentBuilder builder, LeafReader leafReader, int docId) throws IOException { + if (value != null) { + builder.field(name(), value); + } + } + /** * Builder for the binary field mapper * diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 3e96f7651aece..53f6426a6e728 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -226,6 +226,38 @@ private static DateFieldMapper toType(FieldMapper in) { return (DateFieldMapper) in; } + @Override + protected void canDeriveSourceInternal() { + checkStoredAndDocValuesForDerivedSource(); + } + + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + *

+ * Considerations: + * 1. When building source using doc_values, for multi-value field, it will result values in sorted order + *

+ * Date format: + * 1. If "print_format" specified in field mapping, then derived source will have date in this format + * 2. If multiple date formats are specified in field mapping and "print_format" is not specified then + * derived source will contain date in first date format from "||" separated list of format defined in + * "format" + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) { + @Override + public Object convert(Object value) { + Long val = (Long) value; + if (val == null) { + return null; + } + return fieldType().dateTimeFormatter().format(resolution.toInstant(val).atZone(ZoneOffset.UTC)); + } + }, new StoredFieldFetcher(mappedFieldType, simpleName())); + } + /** * Builder for the date field mapper * diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldGenerator.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldGenerator.java new file mode 100644 index 0000000000000..383bd25dc7d0c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldGenerator.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * DerivedSourceGenerator is used to generate derived source field based on field mapping and how + * it is stored in lucene + */ +public class DerivedFieldGenerator { + + private final MappedFieldType mappedFieldType; + private final FieldValueFetcher fieldValueFetcher; + + public DerivedFieldGenerator( + MappedFieldType mappedFieldType, + FieldValueFetcher docValuesFetcher, + FieldValueFetcher storedFieldFetcher + ) { + this.mappedFieldType = mappedFieldType; + if (Objects.requireNonNull(getDerivedFieldPreference()) == FieldValueType.DOC_VALUES) { + assert docValuesFetcher != null; + this.fieldValueFetcher = docValuesFetcher; + } else { + assert storedFieldFetcher != null; + this.fieldValueFetcher = storedFieldFetcher; + } + } + + /** + * Get the preference of the derived field based on field mapping, should be overridden at a FieldMapper to + * alter the preference of derived field + */ + public FieldValueType getDerivedFieldPreference() { + if (mappedFieldType.hasDocValues()) { + return FieldValueType.DOC_VALUES; + } + return FieldValueType.STORED; + } + + /** + * Generate the derived field value based on the preference of derived field and field value type + * @param builder - builder to store the derived source filed + * @param reader - leafReader to read data from + * @param docId - docId for which we want to generate the source + */ + public void generate(XContentBuilder builder, LeafReader reader, int docId) throws IOException { + fieldValueFetcher.write(builder, fieldValueFetcher.fetch(reader, docId)); + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index 4e495c68fd822..f555137cb4f3b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -35,6 +35,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReader; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -213,6 +214,7 @@ public T meta(Map meta) { protected MappedFieldType mappedFieldType; protected MultiFields multiFields; protected CopyTo copyTo; + protected DerivedFieldGenerator derivedFieldGenerator; protected FieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { super(simpleName); @@ -224,6 +226,7 @@ protected FieldMapper(String simpleName, FieldType fieldType, MappedFieldType ma this.mappedFieldType = mappedFieldType; this.multiFields = multiFields; this.copyTo = Objects.requireNonNull(copyTo); + this.derivedFieldGenerator = derivedFieldGenerator(); } @Override @@ -571,6 +574,91 @@ protected static String indexOptionToString(IndexOptions indexOption) { protected abstract String contentType(); + /** + * Method to create derived source generator for this field mapper, it is illegal to enable the + * derived source feature and not implement this method for a field mapper + */ + protected DerivedFieldGenerator derivedFieldGenerator() { + return null; + } + + protected void setDerivedFieldGenerator(DerivedFieldGenerator derivedFieldGenerator) { + this.derivedFieldGenerator = derivedFieldGenerator; + } + + protected DerivedFieldGenerator getDerivedFieldGenerator() { + return this.derivedFieldGenerator; + } + + /** + * Method to determine, if it is possible to derive source for this field using field mapping parameters. + * DerivedFieldGenerator should be set for which derived source feature is supported, this behaviour can be + * overridden at a Mapper level by implementing this method + */ + public void canDeriveSource() { + if (this.copyTo() != null && !this.copyTo().copyToFields().isEmpty()) { + throw new UnsupportedOperationException("Unable to derive source for fields with copy_to parameter set"); + } + canDeriveSourceInternal(); + if (getDerivedFieldGenerator() == null) { + throw new UnsupportedOperationException( + "Derive source is not supported for field [" + name() + "] with field " + "type [" + fieldType().typeName() + "]" + ); + } + } + + /** + * Must be overridden for each mapper for which derived source feature is supported + */ + protected void canDeriveSourceInternal() { + throw new UnsupportedOperationException( + "Derive source is not supported for field [" + name() + "] with field " + "type [" + fieldType().typeName() + "]" + ); + } + + /** + * Validates if doc values is enabled for a field or not + */ + void checkDocValuesForDerivedSource() { + if (!mappedFieldType.hasDocValues()) { + throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with doc values disabled"); + } + } + + /** + * Validates if stored field is enabled for a field or not + */ + void checkStoredForDerivedSource() { + if (!mappedFieldType.isStored()) { + throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with store disabled"); + } + } + + /** + * Validates if doc_values or stored field is enabled for a field or not + */ + void checkStoredAndDocValuesForDerivedSource() { + if (!mappedFieldType.isStored() && !mappedFieldType.hasDocValues()) { + throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with stored and " + "docValues disabled"); + } + } + + /** + * Method used for deriving source and building it to XContentBuilder object + *

+ * Considerations: + * 1. If "null_value" is defined in field mapping and if ingested doc contains "null" field value then derived + * source will contain "null_value" as a displayed field value instead of null and if "null_value" is not + * defined in mapping then field itself will not be present in derived source + * + * @param builder - builder to store the derived source filed + * @param leafReader - leafReader to read data from + * @param docId - docId for which we want to derive the source + */ + public void deriveSource(XContentBuilder builder, LeafReader leafReader, int docId) throws IOException { + derivedFieldGenerator.generate(builder, leafReader, docId); + } + /** * Multi field implementation used across field mappers * diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/FieldValueFetcher.java new file mode 100644 index 0000000000000..aeacf235591ad --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/FieldValueFetcher.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +/** + * Base class for all field value fetchers be it doc values or stored field, consumer should override + * fetch method to read the field value from the LeafReader, read access pattern can be different for + * among different type of doc values as well, like SortedNumericDocValues and SortedSetDocValues + * are stored in different manner, so reading them would also differ + */ +public abstract class FieldValueFetcher { + MappedFieldType mappedFieldType; + final String simpleName; + + protected FieldValueFetcher(String simpleName) { + this.simpleName = simpleName; + } + + /** + * Fetches the field value from the LeafReader, whether it is doc value or stored field + * It should be overridden by fetchers to read the doc values and stored field appropriately + * @param reader - LeafReader to read data from + * @param docId - document id to read + */ + public abstract List fetch(LeafReader reader, int docId) throws IOException; + + /** + * Converts the field value to required representation, should be overridden by field mappers as needed + * @param value - value to convert + */ + Object convert(Object value) { + return value; + } + + /** + * Writes the field value(s) to the builder + * It calls clear() to empty the list containing values after writing them to builder + * For each value, it calls convert to transform the field value to required representation + * @param builder - builder to store the field value(s) in + */ + void write(XContentBuilder builder, List values) throws IOException { + if (values.isEmpty()) { + return; + } + if (values.size() == 1) { + builder.field(simpleName, convert(values.getFirst())); + } else { + final Object[] displayValues = new Object[values.size()]; + for (int i = 0; i < values.size(); i++) { + displayValues[i] = convert(values.get(i)); + } + builder.array(simpleName, displayValues); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldValueType.java b/server/src/main/java/org/opensearch/index/mapper/FieldValueType.java new file mode 100644 index 0000000000000..3d38b8a67e723 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/FieldValueType.java @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +/** + * Indicates the type of field value + */ +public enum FieldValueType { + DOC_VALUES, + STORED +} diff --git a/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java index fcca7e9804bf3..2910bd2856d2f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/GeoPointFieldMapper.java @@ -35,6 +35,7 @@ import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.document.StoredField; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; @@ -199,6 +200,51 @@ public GeoPointFieldType fieldType() { return (GeoPointFieldType) mappedFieldType; } + @Override + protected void canDeriveSourceInternal() { + checkStoredAndDocValuesForDerivedSource(); + } + + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + *

+ * Considerations: + * 1. Result would be returned in fixed format: {"lat", lat_val, "lon": lon_val} for both doc values and + * stored field + * 2. When using doc values, they are indexed in lucene with some loss of precision from the original + * {@code double} values (4.190951585769653E-8 for the latitude component and 8.381903171539307E-8 for + * longitude, same range of precision loss will be there in derived source result). + * 3. When using doc values, for multi value field, result would be in sorted order + * 4. When using stored field, order and duplicate values would be preserved + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) { + @Override + public Object convert(Object value) { + Long val = (Long) value; + if (val == null) { + return null; + } + long encoded = val; + return new GeoPoint( + GeoEncodingUtils.decodeLatitude((int) (encoded >> 32)), + GeoEncodingUtils.decodeLongitude((int) (encoded)) + ); + } + }, new StoredFieldFetcher(mappedFieldType, simpleName()) { + @Override + public Object convert(Object value) { + String val = (String) value; + if (val == null) { + return null; + } + return new GeoPoint(val); + } + }); + } + /** * Concrete field type for geo_point * diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 9fe5d3f240e2c..59bbc73844642 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -178,6 +178,28 @@ public Optional getSupportedDataCubeDimensionType() { return new Builder(n, ignoreMalformedByDefault, c.indexVersionCreated()); }); + @Override + protected void canDeriveSourceInternal() { + checkStoredAndDocValuesForDerivedSource(); + } + + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + *

+ * Considerations: + * 1. When using doc values, for multi value field, result would be deduplicated and in sorted order + * 2. When using stored field, order and duplicate values would be preserved + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator( + mappedFieldType, + new SortedSetDocValuesFetcher(mappedFieldType, simpleName()), + new StoredFieldFetcher(mappedFieldType, simpleName()) + ); + } + /** * Field type for IP fields * diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 2f1160f6a2a6c..1c534153f2629 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -269,6 +269,39 @@ public Optional getSupportedDataCubeDimensionType() { public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers())); + @Override + protected void canDeriveSourceInternal() { + if (this.ignoreAbove != Integer.MAX_VALUE || !Objects.equals(this.normalizerName, "default")) { + throw new UnsupportedOperationException( + "Unable to derive source for [" + name() + "] with " + "ignore_above and/or normalizer set" + ); + } + checkStoredAndDocValuesForDerivedSource(); + } + + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + *

+ * Support: + * 1. If "ignore_above" is set in the field mapping, then we won't be supporting derived source for now, + * considering for these cases we will need to have explicit stored field. + * 2. If "normalizer" is set in the field mapping, then also we won't support derived source, as with + * normalizer it is hard to regenerate original source + *

+ * Considerations: + * 1. When using doc values, for multi value field, result would be deduplicated and in sorted order + * 2. When using stored field, order and duplicate values would be preserved + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator( + mappedFieldType, + new SortedSetDocValuesFetcher(mappedFieldType, simpleName()), + new StoredFieldFetcher(mappedFieldType, simpleName()) + ); + } + /** * Field type for keyword fields * diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 1112734247d0e..61a5308fc3b3c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -59,6 +59,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.index.compositeindex.datacube.DimensionType; @@ -188,6 +189,63 @@ public boolean isDataCubeMetricSupported() { } } + /** + * 1. If it has doc values, build source using doc values + * 2. If doc_values is disabled in field mapping, then build source using stored field + *

+ * Considerations: + * 1. When using doc values, for multi value field, result would be in sorted order + * 2. For "half_float", there might be precision loss, when using doc values(stored as HalfFloatPoint) as + * compared to stored field(stored as float) + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()) { + @Override + public Object convert(Object value) { + Long val = (Long) value; + if (val == null) { + return null; + } + return switch (type) { + case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(val.shortValue()); + case FLOAT -> NumericUtils.sortableIntToFloat(val.intValue()); + case DOUBLE -> NumericUtils.sortableLongToDouble(val); + case BYTE, SHORT, INTEGER, LONG -> val; + case UNSIGNED_LONG -> Numbers.toUnsignedBigInteger(val); + }; + } + + // Unsigned long is sorted according to it's long value, as it is getting ingested as long, so we need to + // sort it again as per its unsigned long value to keep the behavior consistent + @Override + public void write(XContentBuilder builder, List values) throws IOException { + if (type != NumberType.UNSIGNED_LONG) { + super.write(builder, values); + return; + } + if (values.isEmpty()) { + return; + } + if (values.size() == 1) { + builder.field(simpleName, convert(values.getFirst())); + } else { + final BigInteger[] displayValues = new BigInteger[values.size()]; + for (int i = 0; i < values.size(); i++) { + displayValues[i] = (BigInteger) convert(values.get(i)); + } + Arrays.sort(displayValues); + builder.array(simpleName, displayValues); + } + } + }, new StoredFieldFetcher(mappedFieldType, simpleName())); + } + + @Override + protected void canDeriveSourceInternal() { + checkStoredAndDocValuesForDerivedSource(); + } + /** * Type of number * diff --git a/server/src/main/java/org/opensearch/index/mapper/SortedNumericDocValuesFetcher.java b/server/src/main/java/org/opensearch/index/mapper/SortedNumericDocValuesFetcher.java new file mode 100644 index 0000000000000..2867177c2dd4a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/SortedNumericDocValuesFetcher.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * FieldValueFetcher for sorted numeric doc values, for a doc, values will be stored in + * sorted order in lucene. + * + * @opensearch.internal + */ +public class SortedNumericDocValuesFetcher extends FieldValueFetcher { + + public SortedNumericDocValuesFetcher(MappedFieldType mappedFieldType, String SimpleName) { + super(SimpleName); + this.mappedFieldType = mappedFieldType; + } + + @Override + public List fetch(LeafReader reader, int docId) throws IOException { + List values = new ArrayList<>(); + try { + final SortedNumericDocValues sortedNumericDocValues = reader.getSortedNumericDocValues(mappedFieldType.name()); + if (sortedNumericDocValues == null || !sortedNumericDocValues.advanceExact(docId)) { + return values; + } + for (int i = 0; i < sortedNumericDocValues.docValueCount(); i++) { + values.add(sortedNumericDocValues.nextValue()); + } + } catch (Exception e) { + throw new IOException("Failed to read doc values for document " + docId + " in field " + mappedFieldType.name(), e); + } + return values; + } + + @Override + public Object convert(Object value) { + return mappedFieldType.valueForDisplay(value); + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/SortedSetDocValuesFetcher.java b/server/src/main/java/org/opensearch/index/mapper/SortedSetDocValuesFetcher.java new file mode 100644 index 0000000000000..db0b0ab4a95da --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/SortedSetDocValuesFetcher.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * FieldValueFetcher for sorted set doc values, for a doc, values will be deduplicated and sorted while stored in + * lucene + * + * @opensearch.internal + */ +public class SortedSetDocValuesFetcher extends FieldValueFetcher { + + public SortedSetDocValuesFetcher(MappedFieldType mappedFieldType, String simpleName) { + super(simpleName); + this.mappedFieldType = mappedFieldType; + } + + @Override + public List fetch(LeafReader reader, int docId) throws IOException { + List values = new ArrayList<>(); + try { + final SortedSetDocValues sortedSetDocValues = reader.getSortedSetDocValues(mappedFieldType.name()); + if (sortedSetDocValues == null || !sortedSetDocValues.advanceExact(docId)) { + return values; + } + int valueCount = sortedSetDocValues.docValueCount(); + // docValueCount() is equivalent to one plus the maximum ordinal, that means ordinal + // range is [0, docValueCount() - 1] + for (int ord = 0; ord < valueCount; ord++) { + BytesRef value = sortedSetDocValues.lookupOrd(ord); + values.add(BytesRef.deepCopyOf(value)); + } + } catch (IOException e) { + throw new IOException("Failed to read doc values for document " + docId + " in field " + mappedFieldType.name(), e); + } + return values; + } + + @Override + public Object convert(Object value) { + return mappedFieldType.valueForDisplay(value); + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/StoredFieldFetcher.java b/server/src/main/java/org/opensearch/index/mapper/StoredFieldFetcher.java new file mode 100644 index 0000000000000..341405acb2f3d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/StoredFieldFetcher.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.opensearch.index.fieldvisitor.SingleFieldsVisitor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * FieldValueFetcher for stored fields, it uses {@link SingleFieldsVisitor} to fetch the + * field value(s) from stored fields which supports all kind of primitive types + * + * @opensearch.internal + */ +public class StoredFieldFetcher extends FieldValueFetcher { + + public StoredFieldFetcher(MappedFieldType mappedFieldType, String SimpleName) { + super(SimpleName); + this.mappedFieldType = mappedFieldType; + } + + @Override + public List fetch(LeafReader reader, int docId) throws IOException { + List values = new ArrayList<>(); + final SingleFieldsVisitor singleFieldsVisitor = new SingleFieldsVisitor(mappedFieldType, values); + reader.storedFields().document(docId, singleFieldsVisitor); + return values; + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java index df7138863504e..f624ef45544d2 100644 --- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java @@ -1226,4 +1226,28 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, mapperBuilder.indexPrefixes.toXContent(builder, includeDefaults); mapperBuilder.indexPhrases.toXContent(builder, includeDefaults); } + + @Override + protected void canDeriveSourceInternal() { + checkStoredForDerivedSource(); + } + + /** + * 1. Currently, we will only be supporting text field, if stored field is enabled + * + *

+ * Future Improvements + * 1. If there is any subfield present of type keyword, for which source can be derived(doc_values/stored field + * is present and other conditions are meeting for keyword field mapper, i.e. ignore_above or normalizer should + * not be present in subfield mapping) + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, null, new StoredFieldFetcher(mappedFieldType, simpleName())) { + @Override + public FieldValueType getDerivedFieldPreference() { + return FieldValueType.STORED; + } + }; + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 1132c245c6930..21179122c0b5e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -903,4 +903,46 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { private static WildcardFieldMapper toType(FieldMapper in) { return (WildcardFieldMapper) in; } + + @Override + protected void canDeriveSourceInternal() { + if (this.ignoreAbove != Integer.MAX_VALUE || !Objects.equals(this.normalizerName, "default")) { + throw new UnsupportedOperationException( + "Unable to derive source for [" + name() + "] with " + "ignore_above and/or normalizer set" + ); + } + checkDocValuesForDerivedSource(); + } + + /** + * 1. Doc values must be enabled to derive the source, later we can add explicit stored field in case of + * derived source, so that we can always derive source even if doc values are disabled + *

+ * Support: + * 1. If "ignore_above" is set in the field mapping, then we won't be supporting derived source for now, + * considering for these cases we will need to have explicit stored field. + * 2. If "normalizer" is set in the field mapping, then also we won't support derived source, as with + * normalizer it is hard to regenerate original source + *

+ * Considerations: + * 1. When using doc values, for multi value field, result would be deduplicated and in sorted order + */ + @Override + protected DerivedFieldGenerator derivedFieldGenerator() { + return new DerivedFieldGenerator(mappedFieldType, new SortedSetDocValuesFetcher(mappedFieldType, simpleName()) { + @Override + public Object convert(Object value) { + if (value == null) { + return null; + } + BytesRef binaryValue = (BytesRef) value; + return binaryValue.utf8ToString(); + } + }, null) { + @Override + public FieldValueType getDerivedFieldPreference() { + return FieldValueType.DOC_VALUES; + } + }; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 5392bd6c358d3..894e76d0ea442 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -32,14 +32,21 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.Booleans; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -49,6 +56,8 @@ public class BooleanFieldMapperTests extends MapperTestCase { + private static final String FIELD_NAME = "field"; + @Override protected void writeFieldValue(XContentBuilder builder) throws IOException { builder.value(true); @@ -234,4 +243,75 @@ public void testIndexedValueForSearch() throws Exception { assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage()); } + + public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException { + BooleanFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + BooleanFieldMapper mapper = getMapper(copyTo, true, true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + BooleanFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), true, false); + String value = ""; // empty string means false under boolean field mapping + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(value, true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + false + "}", source); + } + } + } + + public void testDerivedValueFetching_StoredField() throws IOException { + try (Directory directory = newDirectory()) { + BooleanFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, true); + String value = "true"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(value, false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + true + "}", source); + } + } + } + + private BooleanFieldMapper getMapper(FieldMapper.CopyTo copyTo, boolean hasDocValues, boolean isStored) throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "boolean").field("store", isStored).field("doc_values", hasDocValues)) + ); + BooleanFieldMapper mapper = (BooleanFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private org.apache.lucene.document.Document createDocument(String value, boolean hasDocValues) { + char[] chars = value.toCharArray(); + boolean val = Booleans.parseBoolean(chars, 0, chars.length, false); + org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document(); + if (hasDocValues) { + doc.add(new SortedNumericDocValuesField(FIELD_NAME, val ? 1 : 0)); + } else { + doc.add(new StoredField(FIELD_NAME, val ? "T" : "F")); + } + return doc; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java index e9d0b6d826ade..ec670ec969bad 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java @@ -8,10 +8,17 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.Directory; import org.opensearch.OpenSearchParseException; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; @@ -30,6 +37,8 @@ public class ConstantKeywordFieldMapperTests extends OpenSearchSingleNodeTestCase { + private static final String FIELD_NAME = "field"; + private IndexService indexService; private DocumentMapperParser parser; @@ -119,4 +128,40 @@ private final SourceToParse source(CheckedConsumer builder.endObject(); return new SourceToParse("test", "1", BytesReference.bytes(builder), MediaTypeRegistry.JSON); } + + public void testPossibleToDeriveSource_WhenCopyToPresent() { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + ConstantKeywordFieldMapper mapper = getMapper(copyTo); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching() throws IOException { + try (Directory directory = newDirectory()) { + ConstantKeywordFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty()); + + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + Document doc = new Document(); + doc.add(new StoredField(FIELD_NAME, "default_value")); + iw.addDocument(doc); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + "\"default_value\"" + "}", source); + } + } + } + + private ConstantKeywordFieldMapper getMapper(FieldMapper.CopyTo copyTo) { + indexService = createIndex("test-index", Settings.EMPTY, "constant_keyword", "field", "type=constant_keyword,value=default_value"); + ConstantKeywordFieldMapper mapper = (ConstantKeywordFieldMapper) indexService.mapperService() + .documentMapper() + .mappers() + .getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java index 9032e2cdaed16..dcd9ef438dd03 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java @@ -32,11 +32,22 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.fieldvisitor.SingleFieldsVisitor; import org.opensearch.index.termvectors.TermVectorsService; import org.opensearch.search.DocValueFormat; @@ -44,15 +55,23 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.Collections; import java.util.List; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assume.assumeThat; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class DateFieldMapperTests extends MapperTestCase { + private static final long TEST_TIMESTAMP = 1739858400000L; + @Override protected void writeFieldValue(XContentBuilder builder) throws IOException { builder.value("2016-03-11"); @@ -343,4 +362,257 @@ public void testFetchDocValuesNanos() throws IOException { assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); } + + public void testPossibleToDeriveSource_WhenDerivedSourceDisabled() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date_nanos").field("format", "strict_date_time||epoch_millis").field("copy_to", "a")) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThrows(UnsupportedOperationException.class, dateFieldMapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date_nanos").field("format", "strict_date_time||epoch_millis").field("copy_to", "a")) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThrows(UnsupportedOperationException.class, dateFieldMapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenDocValuesAndStoreFieldDisabled() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date").field("doc_values", false).field("store", false)) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThrows(UnsupportedOperationException.class, dateFieldMapper::canDeriveSource); + } + + public void testDeriveSource_WhenStoredFieldEnabledAndDateType() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date").field("format", "strict_date_time_no_millis").field("doc_values", false).field("store", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + + StoredFields storedFields = mock(StoredFields.class); + when(leafReader.storedFields()).thenReturn(storedFields); + + FieldInfo mockFieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + DocValuesSkipIndexType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + + doAnswer(invocation -> { + SingleFieldsVisitor visitor = invocation.getArgument(1); + visitor.longField(mockFieldInfo, TEST_TIMESTAMP); + return null; + }).when(storedFields).document(anyInt(), any(StoredFieldVisitor.class)); + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertTrue(source.contains("\"field\":\"2025-02-18T06:00:00Z\"")); + } + + public void testDeriveSource_WhenStoredFieldEnabledAndDateNanosType() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date_nanos") + .field("format", "strict_date_time_no_millis") + .field("doc_values", false) + .field("store", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + + StoredFields storedFields = mock(StoredFields.class); + when(leafReader.storedFields()).thenReturn(storedFields); + + FieldInfo mockFieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + DocValuesSkipIndexType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + + doAnswer(invocation -> { + SingleFieldsVisitor visitor = invocation.getArgument(1); + visitor.longField(mockFieldInfo, TEST_TIMESTAMP * 1000000L); + return null; + }).when(storedFields).document(anyInt(), any(StoredFieldVisitor.class)); + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertTrue(source.contains("\"field\":\"2025-02-18T06:00:00Z\"")); + } + + public void testDeriveSource_WhenStoredFieldEnabledWithMultiValue() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date").field("format", "strict_date_time_no_millis").field("doc_values", false).field("store", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + StoredFields storedFields = mock(StoredFields.class); + when(leafReader.storedFields()).thenReturn(storedFields); + + FieldInfo mockFieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + DocValuesSkipIndexType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + doAnswer(invocation -> { + SingleFieldsVisitor visitor = invocation.getArgument(1); + visitor.longField(mockFieldInfo, TEST_TIMESTAMP); + visitor.longField(mockFieldInfo, TEST_TIMESTAMP + 3600000); // One hour later + return null; + }).when(storedFields).document(anyInt(), any(StoredFieldVisitor.class)); + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertTrue(source.contains("\"field\":[\"2025-02-18T06:00:00Z\",\"2025-02-18T07:00:00Z\"]")); + } + + public void testDeriveSource_WhenDocValuesEnabledAndDateType() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date").field("format", "strict_date_time_no_millis").field("store", false).field("doc_values", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + + SortedNumericDocValues docValues = mock(SortedNumericDocValues.class); + when(leafReader.getSortedNumericDocValues("field")).thenReturn(docValues); + when(docValues.advanceExact(0)).thenReturn(true); + when(docValues.docValueCount()).thenReturn(1); + when(docValues.nextValue()).thenReturn(TEST_TIMESTAMP); + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertTrue(source.contains("\"field\":\"2025-02-18T06:00:00Z\"")); + } + + public void testDeriveSource_WhenDocValuesEnabledAndDateNanosType() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date_nanos") + .field("format", "strict_date_time_no_millis") + .field("store", false) + .field("doc_values", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + + SortedNumericDocValues docValues = mock(SortedNumericDocValues.class); + when(leafReader.getSortedNumericDocValues("field")).thenReturn(docValues); + when(docValues.advanceExact(0)).thenReturn(true); + when(docValues.docValueCount()).thenReturn(1); + when(docValues.nextValue()).thenReturn(TEST_TIMESTAMP * 1000000L); + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertTrue(source.contains("\"field\":\"2025-02-18T06:00:00Z\"")); + } + + public void testDeriveSource_WhenDocValuesEnabledWithMultiValue() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date").field("format", "strict_date_time_no_millis").field("store", false).field("doc_values", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + + SortedNumericDocValues docValues = mock(SortedNumericDocValues.class); + when(leafReader.getSortedNumericDocValues("field")).thenReturn(docValues); + when(docValues.advanceExact(0)).thenReturn(true); + when(docValues.docValueCount()).thenReturn(2); + when(docValues.nextValue()).thenReturn(TEST_TIMESTAMP).thenReturn(TEST_TIMESTAMP + 3600000L); // One Hour Later + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertTrue(source.contains("\"field\":[\"2025-02-18T06:00:00Z\",\"2025-02-18T07:00:00Z\"]")); + } + + public void testDeriveSource_NoValue() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "date").field("format", "strict_date_time_no_millis").field("store", false).field("doc_values", true) + ) + ); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + LeafReader leafReader = mock(LeafReader.class); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + + SortedNumericDocValues docValues = mock(SortedNumericDocValues.class); + when(leafReader.getSortedNumericDocValues("field")).thenReturn(docValues); + when(docValues.advanceExact(0)).thenReturn(false); + + dateFieldMapper.deriveSource(builder, leafReader, 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{}", source); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java index cbb5fc8ce5a22..5538e22c3411c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/GeoPointFieldMapperTests.java @@ -31,14 +31,22 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.geo.GeoUtils; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.hamcrest.CoreMatchers; import java.io.IOException; +import java.util.List; +import java.util.Map; import java.util.Set; import static org.opensearch.geometry.utils.Geohash.stringEncode; @@ -54,6 +62,8 @@ public class GeoPointFieldMapperTests extends FieldMapperTestCase2 { + private static final String FIELD_NAME = "field"; + @Override protected Set unsupportedProperties() { return Set.of("analyzer", "similarity", "doc_values"); @@ -409,4 +419,255 @@ public void testGeoJsonIgnoreInvalidForm() throws Exception { protected GeoPointFieldMapper.Builder newBuilder() { return new GeoPointFieldMapper.Builder("geo"); } + + public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException { + GeoPointFieldMapper mapper = getMapper(getMapperService(false, false), FieldMapper.CopyTo.empty()); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + GeoPointFieldMapper mapper = getMapper(getMapperService(true, false), copyTo); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_DocValues_GeoHash() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, stringEncode(1.3, 1.2)))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_StoredField_GeoHash() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, stringEncode(1.3, 1.2)))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_DocValues_Point() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper() + .parse(source(b -> b.startObject(FIELD_NAME).field("lat", 1.2).field("lon", 1.3).endObject())); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_StoredField_Point() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper() + .parse(source(b -> b.startObject(FIELD_NAME).field("lat", 1.2).field("lon", 1.3).endObject())); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_DocValues_String() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, "1.2,1.3"))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_StoredField_String() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, "1.2,1.3"))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_DocValues_Array() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, new double[] { 1.3, 1.2 }))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_StoredField_Array() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, new double[] { 1.3, 1.2 }))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_DocValues_WKT() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, "POINT (1.3 1.2)"))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_StoredField_WKT() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(FIELD_NAME, "POINT (1.3 1.2)"))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_DocValues_Coordinates() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper() + .parse( + source( + b -> b.startObject(FIELD_NAME) + .field("type", "Point") + .field("coordinates", new double[] { 1.3, 1.2 }) + .endObject() + ) + ); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_StoredField_Coordinates() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper() + .parse( + source( + b -> b.startObject(FIELD_NAME) + .field("type", "Point") + .field("coordinates", new double[] { 1.3, 1.2 }) + .endObject() + ) + ); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSource(mapper, directory); + } + } + + public void testDerivedValueFetching_DocValues_Multi() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(true, false); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper() + .parse(source(b -> b.field(FIELD_NAME, new String[] { stringEncode(1.3, 1.2), stringEncode(1.2, 1.1) }))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSourceMultiField(mapper, directory, false); + } + } + + public void testDerivedValueFetching_StoredField_Multi() throws IOException { + try (Directory directory = newDirectory()) { + MapperService mapperService = getMapperService(false, true); // doc values + GeoPointFieldMapper mapper = getMapper(mapperService, FieldMapper.CopyTo.empty()); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + ParsedDocument doc = mapperService.documentMapper() + .parse(source(b -> b.field(FIELD_NAME, new String[] { stringEncode(1.3, 1.2), stringEncode(1.2, 1.1) }))); + iw.addDocument(doc.rootDoc()); + } + validateDerivedSourceMultiField(mapper, directory, true); + } + } + + private void validateDerivedSource(GeoPointFieldMapper mapper, Directory directory) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertTrue(jsonObject.containsKey(FIELD_NAME)); + Map latLon = (Map) jsonObject.get(FIELD_NAME); + assertEquals(1.2, (Double) latLon.get("lat"), 0.001); + assertEquals(1.3, (Double) latLon.get("lon"), 0.001); + } + } + + private void validateDerivedSourceMultiField(GeoPointFieldMapper mapper, Directory directory, boolean isStored) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertTrue(jsonObject.containsKey(FIELD_NAME)); + List> points = (List>) jsonObject.get(FIELD_NAME); + assertEquals(2, points.size()); + if (isStored) { + Map latLon1 = points.get(0); + assertEquals(1.2, (Double) latLon1.get("lat"), 0.001); + assertEquals(1.3, (Double) latLon1.get("lon"), 0.001); + Map latLon2 = points.get(1); + assertEquals(1.1, (Double) latLon2.get("lat"), 0.001); + assertEquals(1.2, (Double) latLon2.get("lon"), 0.001); + } else { + Map latLon1 = points.get(0); + assertEquals(1.1, (Double) latLon1.get("lat"), 0.001); + assertEquals(1.2, (Double) latLon1.get("lon"), 0.001); + Map latLon2 = points.get(1); + assertEquals(1.2, (Double) latLon2.get("lat"), 0.001); + assertEquals(1.3, (Double) latLon2.get("lon"), 0.001); + } + } + } + + private MapperService getMapperService(boolean hasDocValues, boolean isStored) throws IOException { + return createMapperService( + fieldMapping(b -> b.field("type", "geo_point").field("store", isStored).field("doc_values", hasDocValues)) + ); + } + + private GeoPointFieldMapper getMapper(MapperService mapperService, FieldMapper.CopyTo copyTo) throws IOException { + GeoPointFieldMapper mapper = (GeoPointFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java index 333f151997814..e9bbda4c4471b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java @@ -32,14 +32,22 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.Document; import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.termvectors.TermVectorsService; @@ -50,6 +58,8 @@ public class IpFieldMapperTests extends MapperTestCase { + private static final String FIELD_NAME = "field"; + @Override protected void writeFieldValue(XContentBuilder builder) throws IOException { builder.value("::1"); @@ -208,4 +218,74 @@ public void testNullValue() throws IOException { })); assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored"); } + + public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException { + IpFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + IpFieldMapper mapper = getMapper(copyTo, true, true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + IpFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), true, false); + String ip = "1.2.3.4"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(ip, true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + "\"" + ip + "\"" + "}", source); + } + } + } + + public void testDerivedValueFetching_StoredField() throws IOException { + try (Directory directory = newDirectory()) { + IpFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false, true); + String ip = "1.2.3.4"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(ip, false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + "\"" + ip + "\"" + "}", source); + } + } + } + + private IpFieldMapper getMapper(FieldMapper.CopyTo copyTo, boolean hasDocValues, boolean isStored) throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "ip").field("store", isStored).field("doc_values", hasDocValues)) + ); + IpFieldMapper mapper = (IpFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private Document createDocument(String value, boolean hasDocValues) { + InetAddress address = InetAddresses.forString(value); + Document doc = new Document(); + if (hasDocValues) { + doc.add(new SortedSetDocValuesField(FIELD_NAME, new BytesRef(InetAddressPoint.encode(address)))); + } else { + doc.add(new StoredField(FIELD_NAME, new BytesRef(InetAddressPoint.encode(address)))); + } + return doc; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java index 4da21da40e0d8..3a623f46101de 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java @@ -37,13 +37,21 @@ import org.apache.lucene.analysis.core.LowerCaseFilter; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockLowerCaseFilter; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; @@ -68,6 +76,7 @@ import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; +import static org.opensearch.index.mapper.KeywordFieldMapper.normalizeValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -75,6 +84,8 @@ public class KeywordFieldMapperTests extends MapperTestCase { + private static final String FIELD_NAME = "field"; + /** * Creates a copy of the lowercase token filter which we use for testing merge errors. */ @@ -474,4 +485,101 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] { "hello world" } ); } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + KeywordFieldMapper mapper = getMapper(copyTo, Integer.MAX_VALUE, "default", true, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenIgnoreAbovePresent() throws IOException { + KeywordFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), 100, "default", true, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenNormalizerPresent() throws IOException { + KeywordFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), 100, "lowercase", true, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException { + KeywordFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), Integer.MAX_VALUE, "default", false, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + KeywordFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), Integer.MAX_VALUE, "default", true, false); + String value = "keyword_value"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(mapper, value, true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + "\"" + value + "\"" + "}", source); + } + } + } + + public void testDerivedValueFetching_StoredField() throws IOException { + try (Directory directory = newDirectory()) { + KeywordFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), Integer.MAX_VALUE, "default", false, true); + String value = "keyword_value"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(mapper, value, false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + "\"" + value + "\"" + "}", source); + } + } + } + + private KeywordFieldMapper getMapper( + FieldMapper.CopyTo copyTo, + int ignoreAbove, + String normalizerName, + boolean hasDocValues, + boolean isStored + ) throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "keyword") + .field("store", isStored) + .field("doc_values", hasDocValues) + .field("normalizer", normalizerName) + .field("ignore_above", ignoreAbove) + ) + ); + KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private Document createDocument(KeywordFieldMapper mapper, String value, boolean hasDocValues) throws IOException { + Document doc = new Document(); + FieldType fieldType = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + fieldType.setStored(!hasDocValues); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + NamedAnalyzer normalizer = mapper.fieldType().normalizer(); + value = normalizeValue(normalizer, FIELD_NAME, value); + final BytesRef binaryValue = new BytesRef(value); + if (hasDocValues) { + doc.add(new SortedSetDocValuesField(FIELD_NAME, binaryValue)); + } else { + doc.add(new KeywordFieldMapper.KeywordField(FIELD_NAME, binaryValue, fieldType)); + } + return doc; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java index 610b69a7fdf88..05fe5c8573882 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java @@ -32,8 +32,18 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -52,6 +62,8 @@ public class NumberFieldMapperTests extends AbstractNumericFieldMapperTestCase { + private static final String FIELD_NAME = "field"; + @Override protected Set types() { return Set.of("byte", "short", "integer", "long", "float", "double", "half_float", "unsigned_long"); @@ -319,4 +331,234 @@ public void testLongIndexingOutOfRange() throws Exception { ); assertEquals(0, doc.rootDoc().getFields("field").length); } + + public void testPossibleToDeriveSource_WhenDocValuesAndStoredDisabled() throws IOException { + NumberFieldMapper mapper = getMapper(NumberFieldMapper.NumberType.HALF_FLOAT, FieldMapper.CopyTo.empty(), false, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + NumberFieldMapper mapper = getMapper(NumberFieldMapper.NumberType.HALF_FLOAT, copyTo, true, true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testFloatFieldDerivedValueFetching_DocValues() throws IOException { + NumberType[] floatTypes = { NumberType.FLOAT, NumberType.HALF_FLOAT, NumberType.DOUBLE }; + for (NumberType type : floatTypes) { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(type, FieldMapper.CopyTo.empty(), true, false); + float value = 1.5f; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(type, List.of(value), true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + value + "}", source); + } + } + } + } + + public void testFloatFieldDerivedValueFetching_StoredField() throws IOException { + NumberType[] floatTypes = { NumberType.FLOAT, NumberType.HALF_FLOAT, NumberType.DOUBLE }; + for (NumberType type : floatTypes) { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(type, FieldMapper.CopyTo.empty(), false, true); + float value = 1.5f; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(type, List.of(value), false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + value + "}", source); + } + } + } + } + + public void testIntFieldDerivedValueFetching_DocValues() throws IOException { + NumberType[] fieldTypes = { NumberType.INTEGER, NumberType.SHORT, NumberType.BYTE }; + for (NumberType type : fieldTypes) { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(type, FieldMapper.CopyTo.empty(), true, false); + int value = 123; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(type, List.of(value), true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + value + "}", source); + } + } + } + } + + public void testLongFieldDerivedValueFetching_DocValues() throws IOException { + NumberType[] fieldTypes = { NumberType.LONG, NumberType.UNSIGNED_LONG }; + for (NumberType type : fieldTypes) { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(type, FieldMapper.CopyTo.empty(), true, false); + long value = (1L << 53) + randomLongBetween(0L, 1L << 20); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(type, List.of(value), true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + value + "}", source); + } + } + } + } + + public void testIntFieldDerivedValueFetching_StoredField() throws IOException { + NumberType[] floatTypes = { NumberType.INTEGER, NumberType.LONG, NumberType.UNSIGNED_LONG, NumberType.SHORT, NumberType.BYTE }; + for (NumberType type : floatTypes) { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(type, FieldMapper.CopyTo.empty(), false, true); + int value = 123; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(type, List.of(value), false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + value + "}", source); + } + } + } + } + + public void testLongFieldDerivedValueFetchingMultiValue_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(NumberType.LONG, FieldMapper.CopyTo.empty(), true, false); + long value1 = Integer.MAX_VALUE; + long value2 = Long.MIN_VALUE; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(NumberType.LONG, List.of(value1, value2, value1), true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":[" + value2 + "," + value1 + "," + value1 + "]}", source); + } + } + } + + public void testUnsignedLongFieldDerivedValueFetchingMultiValue_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + NumberFieldMapper mapper = getMapper(NumberType.UNSIGNED_LONG, FieldMapper.CopyTo.empty(), true, false); + long value1 = Integer.MAX_VALUE; + BigInteger value2 = new BigInteger("9223372036854775808"); + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(NumberType.UNSIGNED_LONG, List.of(value2.longValue(), value1, value2.longValue()), true)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":[" + value1 + "," + value2 + "," + value2 + "]}", source); + } + } + } + + private NumberFieldMapper getMapper(NumberType numberType, FieldMapper.CopyTo copyTo, boolean hasDocValues, boolean isStored) + throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", numberType.typeName()).field("store", isStored).field("doc_values", hasDocValues)) + ); + NumberFieldMapper mapper = (NumberFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private Document createDocument(NumberFieldMapper.NumberType type, List values, boolean hasDocValues) { + Document doc = new Document(); + + // Add doc values field + if (hasDocValues) { + for (final Number value : values) { + switch (type) { + case HALF_FLOAT: + doc.add(new SortedNumericDocValuesField(FIELD_NAME, HalfFloatPoint.halfFloatToSortableShort(value.floatValue()))); + break; + case FLOAT: + doc.add(new SortedNumericDocValuesField(FIELD_NAME, NumericUtils.floatToSortableInt(value.floatValue()))); + break; + case DOUBLE: + doc.add(new SortedNumericDocValuesField(FIELD_NAME, NumericUtils.doubleToSortableLong(value.doubleValue()))); + break; + case BYTE: + case SHORT: + case INTEGER: + doc.add(new SortedNumericDocValuesField(FIELD_NAME, value.intValue())); + break; + case LONG: + doc.add(new SortedNumericDocValuesField(FIELD_NAME, value.longValue())); + break; + case UNSIGNED_LONG: + doc.add( + new SortedNumericDocValuesField( + FIELD_NAME, + NumberFieldMapper.NumberType.objectToUnsignedLong(value, false).longValue() + ) + ); + break; + } + } + return doc; + } + + // Add stored field + for (final Number value : values) { + switch (type) { + case HALF_FLOAT: + case FLOAT: + doc.add(new StoredField(FIELD_NAME, value.floatValue())); + break; + case DOUBLE: + doc.add(new StoredField(FIELD_NAME, value.doubleValue())); + break; + case BYTE: + case SHORT: + case INTEGER: + doc.add(new StoredField(FIELD_NAME, value.intValue())); + break; + case LONG: + doc.add(new StoredField(FIELD_NAME, value.longValue())); + break; + case UNSIGNED_LONG: + doc.add(new StoredField(FIELD_NAME, value.toString())); + break; + } + } + return doc; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/SortedNumericDocValuesFetcherTests.java b/server/src/test/java/org/opensearch/index/mapper/SortedNumericDocValuesFetcherTests.java new file mode 100644 index 0000000000000..67ba9d4eff3be --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/SortedNumericDocValuesFetcherTests.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SortedNumericDocValuesFetcherTests extends OpenSearchTestCase { + private MappedFieldType mappedFieldType; + private LeafReader leafReader; + private SortedNumericDocValues sortedNumericDocValues; + + private SortedNumericDocValuesFetcher fetcher; + + @Before + public void setupTest() { + mappedFieldType = mock(MappedFieldType.class); + leafReader = mock(LeafReader.class); + sortedNumericDocValues = mock(SortedNumericDocValues.class); + fetcher = new SortedNumericDocValuesFetcher(mappedFieldType, "test_field"); + } + + public void testFetchSingleValue() throws IOException { + int docId = 1; + long expectedValue = 123L; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedNumericDocValues("test_field")).thenReturn(sortedNumericDocValues); + when(sortedNumericDocValues.advanceExact(docId)).thenReturn(true); + when(sortedNumericDocValues.docValueCount()).thenReturn(1); + when(sortedNumericDocValues.nextValue()).thenReturn(expectedValue); + + List values = fetcher.fetch(leafReader, docId); + + assertEquals(1, values.size()); + assertEquals(expectedValue, values.getFirst()); + } + + public void testFetchMultipleValues() throws IOException { + int docId = 1; + long[] expectedValues = { 1L, 2L, 3L }; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedNumericDocValues("test_field")).thenReturn(sortedNumericDocValues); + when(sortedNumericDocValues.advanceExact(docId)).thenReturn(true); + when(sortedNumericDocValues.docValueCount()).thenReturn(expectedValues.length); + when(sortedNumericDocValues.nextValue()).thenReturn(expectedValues[0]).thenReturn(expectedValues[1]).thenReturn(expectedValues[2]); + + List values = fetcher.fetch(leafReader, docId); + + assertEquals(expectedValues.length, values.size()); + for (int i = 0; i < expectedValues.length; i++) { + assertEquals(expectedValues[i], values.get(i)); + } + } + + public void testFetchNoValues() throws IOException { + int docId = 1; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedNumericDocValues("test_field")).thenReturn(sortedNumericDocValues); + when(sortedNumericDocValues.advanceExact(docId)).thenReturn(true); + when(sortedNumericDocValues.docValueCount()).thenReturn(0); + + List values = fetcher.fetch(leafReader, docId); + + Assert.assertTrue(values.isEmpty()); + } + + public void testConvert() { + Long value = 123L; + String expectedDisplayValue = "123"; + when(mappedFieldType.valueForDisplay(value)).thenReturn(expectedDisplayValue); + + Object result = fetcher.convert(value); + + assertEquals(expectedDisplayValue, result); + verify(mappedFieldType).valueForDisplay(value); + } + + public void testWriteSingleValue() throws IOException { + Long value = 123L; + when(mappedFieldType.name()).thenReturn("test_field"); + when(mappedFieldType.valueForDisplay(value)).thenReturn(value); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + + fetcher.write(builder, List.of(value)); + builder.endObject(); + + String expected = "{\"test_field\":123}"; + assertEquals(expected, builder.toString()); + } + + public void testWriteMultipleValues() throws IOException { + when(mappedFieldType.name()).thenReturn("test_field"); + when(mappedFieldType.valueForDisplay(anyLong())).thenReturn(1, 2, 3); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + + fetcher.write(builder, List.of(1L, 2L, 3L)); + builder.endObject(); + + String expected = "{\"test_field\":[1,2,3]}"; + assertEquals(expected, builder.toString()); + } + + public void testFetchThrowsIOException() throws IOException { + int docId = 1; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedNumericDocValues("test_field")).thenThrow(new IOException("Test exception")); + + assertThrows(IOException.class, () -> fetcher.fetch(leafReader, docId)); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/SortedSetDocValuesFetcherTests.java b/server/src/test/java/org/opensearch/index/mapper/SortedSetDocValuesFetcherTests.java new file mode 100644 index 0000000000000..c83451eedebd3 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/SortedSetDocValuesFetcherTests.java @@ -0,0 +1,139 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.BytesRef; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.DocValueFormat; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SortedSetDocValuesFetcherTests extends OpenSearchTestCase { + private MappedFieldType mappedFieldType; + private LeafReader leafReader; + private SortedSetDocValues sortedSetDocValues; + + private SortedSetDocValuesFetcher fetcher; + + @Before + public void setupTest() { + mappedFieldType = mock(MappedFieldType.class); + leafReader = mock(LeafReader.class); + sortedSetDocValues = mock(SortedSetDocValues.class); + fetcher = new SortedSetDocValuesFetcher(mappedFieldType, "test_field"); + } + + public void testFetchSingleValue() throws IOException { + int docId = 1; + String value = "123"; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedSetDocValues("test_field")).thenReturn(sortedSetDocValues); + when(sortedSetDocValues.advanceExact(docId)).thenReturn(true); + when(sortedSetDocValues.docValueCount()).thenReturn(1); + when(sortedSetDocValues.lookupOrd(anyLong())).thenReturn(new BytesRef(value.getBytes(StandardCharsets.UTF_8))); + + List res = fetcher.fetch(leafReader, docId); + + assertEquals(1, res.size()); + assertEquals(value, DocValueFormat.RAW.format((BytesRef) res.getFirst())); + } + + public void testFetchMultipleValues() throws IOException { + int docId = 1; + String[] values = { "1", "2", "3" }; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedSetDocValues("test_field")).thenReturn(sortedSetDocValues); + when(sortedSetDocValues.advanceExact(docId)).thenReturn(true); + when(sortedSetDocValues.docValueCount()).thenReturn(values.length); + when(sortedSetDocValues.lookupOrd(anyLong())).thenReturn(new BytesRef(values[0].getBytes(StandardCharsets.UTF_8))) + .thenReturn(new BytesRef(values[1].getBytes(StandardCharsets.UTF_8))) + .thenReturn(new BytesRef(values[2].getBytes(StandardCharsets.UTF_8))); + + List res = fetcher.fetch(leafReader, docId); + + assertEquals(values.length, res.size()); + for (int i = 0; i < values.length; i++) { + assertEquals(values[i], DocValueFormat.RAW.format((BytesRef) res.get(i))); + } + } + + public void testFetchNoValues() throws IOException { + int docId = 1; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedSetDocValues("test_field")).thenReturn(sortedSetDocValues); + when(sortedSetDocValues.advanceExact(docId)).thenReturn(true); + when(sortedSetDocValues.docValueCount()).thenReturn(0); + + List res = fetcher.fetch(leafReader, docId); + Assert.assertTrue(res.isEmpty()); + } + + public void testConvert() { + Long value = 123L; + String expectedDisplayValue = "123"; + when(mappedFieldType.valueForDisplay(value)).thenReturn(expectedDisplayValue); + + Object result = fetcher.convert(value); + + assertEquals(expectedDisplayValue, result); + verify(mappedFieldType).valueForDisplay(value); + } + + public void testWriteSingleValue() throws IOException { + Long value = 123L; + List values = new ArrayList<>(); + values.add(value); + when(mappedFieldType.name()).thenReturn("test_field"); + when(mappedFieldType.valueForDisplay(value)).thenReturn(value); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + + fetcher.write(builder, values); + builder.endObject(); + + String expected = "{\"test_field\":123}"; + assertEquals(expected, builder.toString()); + } + + public void testWriteMultipleValues() throws IOException { + List values = List.of(1L, 2L, 3L); + when(mappedFieldType.name()).thenReturn("test_field"); + when(mappedFieldType.valueForDisplay(anyLong())).thenReturn(1, 2, 3); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + + fetcher.write(builder, values); + builder.endObject(); + + String expected = "{\"test_field\":[1,2,3]}"; + assertEquals(expected, builder.toString()); + } + + public void testFetchThrowsIOException() throws IOException { + int docId = 1; + when(mappedFieldType.name()).thenReturn("test_field"); + when(leafReader.getSortedSetDocValues("test_field")).thenThrow(new IOException("Test exception")); + + assertThrows(IOException.class, () -> fetcher.fetch(leafReader, docId)); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/StoredFieldFetcherTests.java b/server/src/test/java/org/opensearch/index/mapper/StoredFieldFetcherTests.java new file mode 100644 index 0000000000000..9cc553eb1b626 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/StoredFieldFetcherTests.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.fieldvisitor.SingleFieldsVisitor; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class StoredFieldFetcherTests extends OpenSearchTestCase { + private StoredFieldFetcher fetcher; + private MappedFieldType mappedFieldType; + private LeafReader leafReader; + private StoredFields storedFields; + + @Before + public void setupTest() { + mappedFieldType = mock(MappedFieldType.class); + leafReader = mock(LeafReader.class); + storedFields = mock(StoredFields.class); + fetcher = new StoredFieldFetcher(mappedFieldType, "test_field"); + } + + public void testFetchStoredField() throws IOException { + int docId = 1; + when(leafReader.storedFields()).thenReturn(storedFields); + doNothing().when(storedFields).document(eq(docId), any(SingleFieldsVisitor.class)); + + fetcher.fetch(leafReader, docId); + + verify(leafReader).storedFields(); + verify(storedFields).document(eq(docId), any(SingleFieldsVisitor.class)); + } + + public void testFetchThrowsIOException() throws IOException { + int docId = 1; + when(leafReader.storedFields()).thenReturn(storedFields); + doThrow(new IOException("Test exception")).when(storedFields).document(eq(docId), any(SingleFieldsVisitor.class)); + + assertThrows(IOException.class, () -> fetcher.fetch(leafReader, docId)); + } + + public void testMultipleFetchCalls() throws IOException { + int docId1 = 1; + int docId2 = 2; + when(leafReader.storedFields()).thenReturn(storedFields); + + fetcher.fetch(leafReader, docId1); + fetcher.fetch(leafReader, docId2); + + verify(leafReader, times(2)).storedFields(); + verify(storedFields).document(eq(docId1), any(SingleFieldsVisitor.class)); + verify(storedFields).document(eq(docId2), any(SingleFieldsVisitor.class)); + } + + public void testNullStoredFields() throws IOException { + when(leafReader.storedFields()).thenReturn(null); + + assertThrows(NullPointerException.class, () -> fetcher.fetch(leafReader, 1)); + } + + public void testWriteSingleValue() throws IOException { + int docId = 1; + when(mappedFieldType.name()).thenReturn("test_field"); + when(mappedFieldType.valueForDisplay("123")).thenReturn("123"); + when(leafReader.storedFields()).thenReturn(storedFields); + FieldInfo mockFieldInfo = new FieldInfo( + "test_field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + DocValuesSkipIndexType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + doAnswer(invocation -> { + SingleFieldsVisitor visitor = invocation.getArgument(1); + visitor.stringField(mockFieldInfo, "123"); + return null; + }).when(storedFields).document(eq(docId), any(StoredFieldVisitor.class)); + + List values = fetcher.fetch(leafReader, docId); + + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + fetcher.write(builder, values); + builder.endObject(); + + String expected = "{\"test_field\":\"123\"}"; + assertEquals(expected, builder.toString()); + } + + public void testWriteMultiValue() throws IOException { + int docId = 1; + when(mappedFieldType.name()).thenReturn("test_field"); + when(mappedFieldType.valueForDisplay(anyString())).thenReturn("1", "2", "3"); + when(leafReader.storedFields()).thenReturn(storedFields); + FieldInfo mockFieldInfo = new FieldInfo( + "test_field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + DocValuesSkipIndexType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + doAnswer(invocation -> { + SingleFieldsVisitor visitor = invocation.getArgument(1); + visitor.stringField(mockFieldInfo, "1"); + visitor.stringField(mockFieldInfo, "2"); + visitor.stringField(mockFieldInfo, "3"); + return null; + }).when(storedFields).document(eq(docId), any(StoredFieldVisitor.class)); + + List values = fetcher.fetch(leafReader, docId); + + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + fetcher.write(builder, values); + builder.endObject(); + + String expected = "{\"test_field\":[\"1\",\"2\",\"3\"]}"; + assertEquals(expected, builder.toString()); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java index 0253caea9759d..d8d7bb1724865 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java @@ -40,9 +40,15 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.PostingsEnum; @@ -59,11 +65,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.MockSynonymAnalyzer; import org.apache.lucene.tests.analysis.Token; import org.apache.lucene.util.BytesRef; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -1066,4 +1074,58 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(TextFieldMapper.class)); assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + TextFieldMapper mapper = getMapper(copyTo, false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenStoredFieldDisabled() throws IOException { + TextFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_StoredField() throws IOException { + try (Directory directory = newDirectory()) { + TextFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), true); + String value = "value"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument("field", value, false, false)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + "field" + "\":" + "\"" + value + "\"" + "}", source); + } + } + } + + private TextFieldMapper getMapper(FieldMapper.CopyTo copyTo, boolean isStored) throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "text").field("store", isStored))); + TextFieldMapper mapper = (TextFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private Document createDocument(String name, String value, boolean forKeyword, boolean hasDocValues) { + Document doc = new Document(); + final BytesRef binaryValue = new BytesRef(value); + if (hasDocValues) { + doc.add(new SortedSetDocValuesField(name, binaryValue)); + } else { + if (forKeyword) { + doc.add(new StoredField(name, binaryValue)); + } else { + doc.add(new StoredField(name, value)); + } + } + return doc; + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java index 25aacb41f029d..d50cd380cd824 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java @@ -14,15 +14,22 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; @@ -43,9 +50,12 @@ import static java.util.Collections.singletonMap; import static org.opensearch.index.mapper.FieldTypeTestCase.fetchSourceValue; +import static org.opensearch.index.mapper.KeywordFieldMapper.normalizeValue; public class WildcardFieldMapperTests extends MapperTestCase { + private static final String FIELD_NAME = "field"; + @Override protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "wildcard"); @@ -313,4 +323,70 @@ public void testFetchSourceValue() throws IOException { MappedFieldType nullValueMapper = new WildcardFieldMapper.Builder("field").nullValue("NULL").build(context).fieldType(); assertEquals(Collections.singletonList("NULL"), fetchSourceValue(nullValueMapper, null)); } + + public void testPossibleToDeriveSource_WhenCopyToPresent() throws IOException { + FieldMapper.CopyTo copyTo = new FieldMapper.CopyTo.Builder().add("copy_to_field").build(); + WildcardFieldMapper mapper = getMapper(copyTo, Integer.MAX_VALUE, "default", true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenIgnoreAbovePresent() throws IOException { + WildcardFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), 100, "default", true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenNormalizerPresent() throws IOException { + WildcardFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), 100, "lowercase", true); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testPossibleToDeriveSource_WhenDocValuesDisabled() throws IOException { + WildcardFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), Integer.MAX_VALUE, "default", false); + assertThrows(UnsupportedOperationException.class, mapper::canDeriveSource); + } + + public void testDerivedValueFetching_DocValues() throws IOException { + try (Directory directory = newDirectory()) { + WildcardFieldMapper mapper = getMapper(FieldMapper.CopyTo.empty(), Integer.MAX_VALUE, "default", true); + String value = "keyword_value"; + try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig())) { + iw.addDocument(createDocument(mapper, value)); + } + + try (DirectoryReader reader = DirectoryReader.open(directory)) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.deriveSource(builder, reader.leaves().get(0).reader(), 0); + builder.endObject(); + String source = builder.toString(); + assertEquals("{\"" + FIELD_NAME + "\":" + "\"" + value + "\"" + "}", source); + } + } + } + + private WildcardFieldMapper getMapper(FieldMapper.CopyTo copyTo, int ignoreAbove, String normalizerName, boolean hasDocValues) + throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "wildcard") + .field("doc_values", hasDocValues) + .field("normalizer", normalizerName) + .field("ignore_above", ignoreAbove) + ) + ); + WildcardFieldMapper mapper = (WildcardFieldMapper) mapperService.documentMapper().mappers().getMapper(FIELD_NAME); + mapper.copyTo = copyTo; + return mapper; + } + + /** + * Helper method to create a document with both doc values and stored fields + */ + private Document createDocument(WildcardFieldMapper mapper, String value) throws IOException { + Document doc = new Document(); + NamedAnalyzer normalizer = mapper.fieldType().normalizer(); + value = normalizeValue(normalizer, FIELD_NAME, value); + final BytesRef binaryValue = new BytesRef(value); + doc.add(new SortedSetDocValuesField(FIELD_NAME, binaryValue)); + return doc; + } }