diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index ee50e19eff040..78675c54b132d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -378,6 +378,11 @@ protected ScaledFloatFieldMapper clone() { return (ScaledFloatFieldMapper) super.clone(); } + @Override + protected Double nullValue() { + return nullValue; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { @@ -502,7 +507,16 @@ protected Double parseSourceValue(Object value, String format) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - double doubleValue = objectToDouble(value); + double doubleValue; + if (value.equals("")) { + if (nullValue == null) { + return null; + } + doubleValue = nullValue; + } else { + doubleValue = objectToDouble(value); + } + double scalingFactor = fieldType().getScalingFactor(); return Math.round(doubleValue * scalingFactor) / scalingFactor; } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index bcbf6fb432f79..6758750f66581 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -405,11 +406,22 @@ public void testMeta() throws Exception { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field") .scalingFactor(100) .build(context); - assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001); assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001); + assertNull(mapper.parseSourceValue("", null)); + + ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field") + .scalingFactor(100) + .nullValue(2.71) + .build(context); + assertEquals(2.71, nullValueMapper.parseSourceValue("", null), 0.00001); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(2.71), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 1aef94981e7ad..ef8d76456a8d3 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -598,6 +598,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { ICUCollationKeywordFieldMapper icuMergeWith = (ICUCollationKeywordFieldMapper) other; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 1784a74d11f35..b809b0fb87139 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -38,12 +38,15 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -489,9 +492,8 @@ public void testUpdateIgnoreAbove() throws IOException { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value", null)); + ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); assertEquals("42", mapper.parseSourceValue(42L, null)); assertEquals("true", mapper.parseSourceValue(true, null)); @@ -501,5 +503,12 @@ public void testParseSourceValue() { assertNull(ignoreAboveMapper.parseSourceValue("value", null)); assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + + ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index 05f8cb3a2fc45..20c3b01ead7b1 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -97,6 +97,16 @@ private static void extractRawValues(List values, List part, String[] pa } } + /** + * For the provided path, return its value in the xContent map. + * + * Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values + * can be returned. + * + * @param path the value's path in the map. + * + * @return the value associated with the path in the map or 'null' if the path does not exist. + */ public static Object extractValue(String path, Map map) { return extractValue(map, path.split("\\.")); } @@ -105,19 +115,51 @@ public static Object extractValue(Map map, String... pathElements) { if (pathElements.length == 0) { return null; } - return extractValue(pathElements, 0, map); + return XContentMapValues.extractValue(pathElements, 0, map, null); } - @SuppressWarnings({"unchecked"}) - private static Object extractValue(String[] pathElements, int index, Object currentValue) { - if (index == pathElements.length) { - return currentValue; - } - if (currentValue == null) { + /** + * For the provided path, return its value in the xContent map. + * + * Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values + * can be returned. + * + * @param path the value's path in the map. + * @param nullValue a value to return if the path exists, but the value is 'null'. This helps + * in distinguishing between a path that doesn't exist vs. a value of 'null'. + * + * @return the value associated with the path in the map or 'null' if the path does not exist. + */ + public static Object extractValue(String path, Map map, Object nullValue) { + String[] pathElements = path.split("\\."); + if (pathElements.length == 0) { return null; } + return extractValue(pathElements, 0, map, nullValue); + } + + private static Object extractValue(String[] pathElements, + int index, + Object currentValue, + Object nullValue) { + if (currentValue instanceof List) { + List valueList = (List) currentValue; + List newList = new ArrayList<>(valueList.size()); + for (Object o : valueList) { + Object listValue = extractValue(pathElements, index, o, nullValue); + if (listValue != null) { + newList.add(listValue); + } + } + return newList; + } + + if (index == pathElements.length) { + return currentValue != null ? currentValue : nullValue; + } + if (currentValue instanceof Map) { - Map map = (Map) currentValue; + Map map = (Map) currentValue; String key = pathElements[index]; Object mapValue = map.get(key); int nextIndex = index + 1; @@ -126,18 +168,12 @@ private static Object extractValue(String[] pathElements, int index, Object curr mapValue = map.get(key); nextIndex++; } - return extractValue(pathElements, nextIndex, mapValue); - } - if (currentValue instanceof List) { - List valueList = (List) currentValue; - List newList = new ArrayList(valueList.size()); - for (Object o : valueList) { - Object listValue = extractValue(pathElements, index, o); - if (listValue != null) { - newList.add(listValue); - } + + if (map.containsKey(key) == false) { + return null; } - return newList; + + return extractValue(pathElements, nextIndex, mapValue, nullValue); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 8786d24a03034..cf9c88c18ac57 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -285,6 +285,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected Object nullValue() { + return nullValue; + } + @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index eeaf91da8c716..032199fd85e58 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -581,6 +581,11 @@ protected DateFieldMapper clone() { return (DateFieldMapper) super.clone(); } + @Override + protected String nullValue() { + return nullValueAsString; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { String dateAsString; @@ -631,8 +636,8 @@ protected void parseCreateField(ParseContext context) throws IOException { public String parseSourceValue(Object value, String format) { String date = value.toString(); long timestamp = fieldType().parse(date); - ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); + ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); if (format != null) { dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index d55f99ebee7a2..bd32a99ea68c4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -223,6 +223,13 @@ public CopyTo copyTo() { return copyTo; } + /** + * A value to use in place of a {@code null} value in the document source. + */ + protected Object nullValue() { + return null; + } + /** * Whether this mapper can handle an array value during document parsing. If true, * when an array is encountered during parsing, the document parser will pass the @@ -285,7 +292,7 @@ public void parse(ParseContext context) throws IOException { * @return a list a standardized field values. */ public List lookupValues(SourceLookup lookup, @Nullable String format) { - Object sourceValue = lookup.extractValue(name()); + Object sourceValue = lookup.extractValue(name(), nullValue()); if (sourceValue == null) { return List.of(); } @@ -337,6 +344,7 @@ protected FieldMapper clone() { } } + @Override public final FieldMapper merge(Mapper mergeWith) { FieldMapper merged = clone(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index f7d5d4aab1e82..3b11de788cd4e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -357,6 +357,11 @@ protected String contentType() { return fieldType().typeName(); } + @Override + protected Object nullValue() { + return nullValue; + } + @Override protected IpFieldMapper clone() { return (IpFieldMapper) super.clone(); @@ -415,7 +420,12 @@ protected String parseSourceValue(Object value, String format) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - InetAddress address = InetAddresses.forString(value.toString()); + InetAddress address; + if (value instanceof InetAddress) { + address = (InetAddress) value; + } else { + address = InetAddresses.forString(value.toString()); + } return InetAddresses.toAddrString(address); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 79932d8c8e45c..a8daf057b0461 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -441,6 +441,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { KeywordFieldMapper k = (KeywordFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 7e20e3e9718fc..fbe1b0beaa631 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1052,6 +1052,11 @@ protected NumberFieldMapper clone() { return (NumberFieldMapper) super.clone(); } + @Override + protected Number nullValue() { + return nullValue; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); @@ -1106,6 +1111,11 @@ protected Number parseSourceValue(Object value, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } + + if (value.equals("")) { + return nullValue; + } + return fieldType().type.parse(value, coerce.value()); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 6393d4bf50ff0..d63caed14adb0 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentHelper; @@ -133,11 +134,19 @@ public List extractRawValues(String path) { } /** - * For the provided path, return its value in the source. Note that in contrast with - * {@link SourceLookup#extractRawValues}, array and object values can be returned. + * For the provided path, return its value in the source. + * + * Note that in contrast with {@link SourceLookup#extractRawValues}, array and object values + * can be returned. + * + * @param path the value's path in the source. + * @param nullValue a value to return if the path exists, but the value is 'null'. This helps + * in distinguishing between a path that doesn't exist vs. a value of 'null'. + * + * @return the value associated with the path in the source or 'null' if the path does not exist. */ - public Object extractValue(String path) { - return XContentMapValues.extractValue(path, loadSourceIfNeeded()); + public Object extractValue(String path, @Nullable Object nullValue) { + return XContentMapValues.extractValue(path, loadSourceIfNeeded(), nullValue); } public Object filter(FetchSourceContext context) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index d83000bd66956..957316d99dad8 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -164,6 +164,35 @@ public void testExtractValue() throws Exception { assertThat(XContentMapValues.extractValue("path1.xxx.path2.yyy.test", map).toString(), equalTo("value")); } + public void testExtractValueWithNullValue() throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .nullField("other_field") + .array("array", "value1", null, "value2") + .startObject("object1") + .startObject("object2").nullField("field").endObject() + .endObject() + .startArray("object_array") + .startObject().nullField("field").endObject() + .startObject().field("field", "value").endObject() + .endArray() + .endObject(); + + Map map; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { + map = parser.map(); + } + assertEquals("value", XContentMapValues.extractValue("field", map, "NULL")); + assertNull(XContentMapValues.extractValue("missing", map, "NULL")); + assertNull(XContentMapValues.extractValue("field.missing", map, "NULL")); + assertNull(XContentMapValues.extractValue("object1.missing", map, "NULL")); + + assertEquals("NULL", XContentMapValues.extractValue("other_field", map, "NULL")); + assertEquals(List.of("value1", "NULL", "value2"), XContentMapValues.extractValue("array", map, "NULL")); + assertEquals(List.of("NULL", "value"), XContentMapValues.extractValue("object_array.field", map, "NULL")); + assertEquals("NULL", XContentMapValues.extractValue("object1.object2.field", map, "NULL")); + } + public void testExtractRawValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field("test", "value") diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 9ecdaa98d0049..314bf7c5bedfc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -44,12 +44,14 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -291,11 +293,18 @@ public void testMeta() throws Exception { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); + BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); assertTrue(mapper.parseSourceValue(true, null)); assertFalse(mapper.parseSourceValue("false", null)); assertFalse(mapper.parseSourceValue("", null)); + + BooleanFieldMapper nullValueMapper = new BooleanFieldMapper.Builder("field") + .nullValue(true) + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(true), nullValueMapper.lookupValues(sourceLookup, null)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index ad6067caf9262..8323d74912df6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -44,6 +45,7 @@ import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Set; @@ -489,6 +491,14 @@ public void testParseSourceValue() { String dateInMillis = "662428800000"; assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis, null)); assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L, null)); + + String nullValueDate = "2020-05-15T21:33:02.000Z"; + DateFieldMapper nullValueMapper = new DateFieldMapper.Builder("field") + .nullValue(nullValueDate) + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(nullValueDate), nullValueMapper.lookupValues(sourceLookup, null)); } public void testParseSourceValueWithFormat() { @@ -497,10 +507,15 @@ public void testParseSourceValueWithFormat() { DateFieldMapper mapper = new DateFieldMapper.Builder("field") .format("strict_date_time") + .nullValue("1970-12-29T00:00:00.000Z") .build(context); String date = "1990-12-29T00:00:00.000Z"; assertEquals("1990/12/29", mapper.parseSourceValue(date, "yyyy/MM/dd")); assertEquals("662428800000", mapper.parseSourceValue(date, "epoch_millis")); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("1970/12/29"), mapper.lookupValues(sourceLookup, "yyyy/MM/dd")); } public void testParseSourceValueNanos() { @@ -514,5 +529,16 @@ public void testParseSourceValueNanos() { String date = "2020-05-15T21:33:02.123456789Z"; assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date, null)); assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L, null)); + + String nullValueDate = "2020-05-15T21:33:02.123456789Z"; + DateFieldMapper nullValueMapper = new DateFieldMapper.Builder("field") + .format("strict_date_time||epoch_millis") + .nullValue(nullValueDate) + .withResolution(DateFieldMapper.Resolution.NANOSECONDS) + .build(context); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(nullValueDate), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 055162c1b0862..405efdecb8248 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -40,12 +40,15 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.net.InetAddress; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -303,11 +306,18 @@ public void testEmptyName() throws IOException { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); + IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1", null)); assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1", null)); assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1", null)); + + IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field") + .nullValue(InetAddresses.forString("2001:db8:0:0:0:0:2:7")) + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("2001:db8::2:7"), nullValueMapper.lookupValues(sourceLookup, null)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 5d088588dd3a6..19552730fcb76 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -643,5 +644,12 @@ public void testParseSourceValue() { assertNull(ignoreAboveMapper.parseSourceValue("value", null)); assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + + KeywordFieldMapper nullValueMapper = new KeywordFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 9f89122c10aa1..c2a60272225bb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -38,11 +38,13 @@ import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.ByteArrayInputStream; import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -407,10 +409,19 @@ public void testEmptyName() throws IOException { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); + NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); assertEquals(3, mapper.parseSourceValue(3.14, null)); assertEquals(42, mapper.parseSourceValue("42.9", null)); + + NumberFieldMapper nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT) + .nullValue(2.71f) + .build(context); + assertEquals(2.71f, (float) nullValueMapper.parseSourceValue("", null), 0.00001); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(2.71f), nullValueMapper.lookupValues(sourceLookup, null)); } @Timeout(millis = 30000) diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index bef137dba6985..2b87e1d770411 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -581,6 +581,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper mergeWith, List conflicts) { FlatObjectFieldMapper other = ((FlatObjectFieldMapper) mergeWith); diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java index 4f9dda15bdf0a..a1be51791aa8d 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java @@ -9,24 +9,30 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapperTestCase; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin; import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType; @@ -36,6 +42,9 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; @@ -506,4 +515,19 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] {"Hello", "World"}); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + Map sourceValue = Map.of("key", "value"); + FlatObjectFieldMapper mapper = new FlatObjectFieldMapper.Builder("field").build(context); + assertEquals(sourceValue, mapper.parseSourceValue(sourceValue, null)); + + FlatObjectFieldMapper nullValueMapper = new FlatObjectFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); + } } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 7d03fcd95da7a..835cee9a3215f 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -1001,6 +1001,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { this.ignoreAbove = ((WildcardFieldMapper) other).ignoreAbove; diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index a3176576c97ec..54e3b9f7757dc 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -55,6 +55,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -64,7 +65,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.function.BiFunction; import static org.hamcrest.Matchers.equalTo; @@ -789,6 +792,13 @@ public void testParseSourceValue() { assertNull(ignoreAboveMapper.parseSourceValue("value", null)); assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + + WildcardFieldMapper nullValueMapper = new WildcardFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); } protected MappedFieldType provideMappedFieldType(String name) {