[fixes #16599] Field-level ignore_malformed should override index-level setting

CHANGELOG.md

@@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640))
 - Fix Replication lag computation ([#18602](https://github.com/opensearch-project/OpenSearch/pull/18602))
 - Fix max_score is null when sorting on score firstly ([#18715](https://github.com/opensearch-project/OpenSearch/pull/18715))
+- Field-level ignore_malformed should override index-level setting ([#18706](https://github.com/opensearch-project/OpenSearch/pull/18706))
 - Fixed Staggered merge -  load average replace with AverageTrackers, some Default thresholds modified ([#18666](https://github.com/opensearch-project/OpenSearch/pull/18666))
 - Use `new SecureRandom()` to avoid blocking ([18729](https://github.com/opensearch-project/OpenSearch/issues/18729))
 

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -395,8 +395,9 @@ boolean coerce() {
         return coerce.value();
     }
 
-    boolean ignoreMalformed() {
-        return ignoreMalformed.value();
+    @Override
+    protected Explicit<Boolean> ignoreMalformed() {
+        return ignoreMalformed;
     }
 
     @Override

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -90,7 +90,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
         checker.registerUpdateCheck(b -> b.field("coerce", false), m -> assertFalse(((ScaledFloatFieldMapper) m).coerce()));
         checker.registerUpdateCheck(
             b -> b.field("ignore_malformed", true),
-            m -> assertTrue(((ScaledFloatFieldMapper) m).ignoreMalformed())
+            m -> assertTrue(((ScaledFloatFieldMapper) m).ignoreMalformed().value())
         );
     }
 

rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml

@@ -2,7 +2,7 @@
 setup:
   - do:
       indices.create:
-          index:  test
+          index: test
           body:
             mappings:
               properties:
@@ -13,6 +13,19 @@ setup:
                 my_ip:
                   type: ip
                   ignore_malformed: true
+  - do:
+      indices.create:
+        index: index_level_setting
+        body:
+          settings:
+            index.mapping.ignore_malformed: true
+          mappings:
+            properties:
+              my_date:
+                type: date
+                ignore_malformed: false
+              my_ip:
+                type: ip
 
   - do:
       index:
@@ -40,6 +53,7 @@ setup:
 
   - do:
       search:
+        index: test
         rest_total_hits_as_int: true
         body: { query: { exists: { "field": "_ignored" } } }
 
@@ -50,6 +64,7 @@ setup:
 
   - do:
       search:
+        index: test
         rest_total_hits_as_int: true
         body: { query: { term: { "_ignored": "my_date" } } }
 
@@ -60,6 +75,7 @@ setup:
 
   - do:
       search:
+        index: test
         rest_total_hits_as_int: true
         body: { query: { terms: { "_ignored": [ "my_date", "my_ip" ] } } }
 
@@ -70,6 +86,7 @@ setup:
 
   - do:
       search:
+        index: test
         rest_total_hits_as_int: true
         body: { query: { ids: { "values": [ "3" ] } } }
 
@@ -81,9 +98,21 @@ setup:
 
   - do:
       search:
+        index: test
         rest_total_hits_as_int: true
         stored_fields: [ "my_date" ]
         body: { query: { ids: { "values": [ "3" ] } } }
 
   - length:   { hits.hits: 1  }
   - is_true:  hits.hits.0._ignored
+
+---
+"field-level setting shall override index-level setting":
+  - skip:
+      version: " - 3.1.99"
+      reason: "Versions pre-3.2.0 are affected by the bug"
+  - do:
+      catch: /failed to parse field \[my_date\] of type \[date\]/
+      index:
+        index: index_level_setting
+        body: { "my_date": "bar", "my_ip": "quux" }

server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -452,7 +452,8 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
         }
     }
 
-    public Explicit<Boolean> ignoreMalformed() {
+    @Override
+    protected Explicit<Boolean> ignoreMalformed() {
         return ignoreMalformed;
     }
 

server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java

@@ -44,6 +44,7 @@
 import org.apache.lucene.search.Query;
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.Version;
+import org.opensearch.common.Explicit;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.geo.ShapeRelation;
 import org.opensearch.common.logging.DeprecationLogger;
@@ -294,7 +295,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
 
         private final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null)
             .acceptsNull();
-        private final Parameter<Boolean> ignoreMalformed;
+        private final Parameter<Explicit<Boolean>> ignoreMalformed;
 
         private final Resolution resolution;
         private final Version indexCreatedVersion;
@@ -309,7 +310,12 @@ public Builder(
             super(name);
             this.resolution = resolution;
             this.indexCreatedVersion = indexCreatedVersion;
-            this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
+            this.ignoreMalformed = Parameter.explicitBoolParam(
+                "ignore_malformed",
+                true,
+                m -> toType(m).ignoreMalformed,
+                ignoreMalformedByDefault
+            );
             if (dateFormatter != null) {
                 this.format.setValue(dateFormatter.pattern());
                 this.printFormat.setValue(dateFormatter.printPattern());
@@ -721,7 +727,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
     private final Locale locale;
     private final String format;
     private final String printFormat;
-    private final boolean ignoreMalformed;
+    private final Explicit<Boolean> ignoreMalformed;
     private final Long nullValue;
     private final String nullValueAsString;
     private final Resolution resolution;
@@ -749,7 +755,7 @@ private DateFieldMapper(
         this.nullValueAsString = builder.nullValue.getValue();
         this.nullValue = nullValue;
         this.resolution = resolution;
-        this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue();
+        this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
         this.indexCreatedVersion = builder.indexCreatedVersion;
     }
 
@@ -797,7 +803,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
             try {
                 timestamp = fieldType().parse(dateAsString);
             } catch (IllegalArgumentException | OpenSearchParseException | DateTimeException | ArithmeticException e) {
-                if (ignoreMalformed) {
+                if (ignoreMalformed().value()) {
                     context.addIgnoredField(mappedFieldType.name());
                     return;
                 } else {
@@ -819,11 +825,12 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
     }
 
-    public boolean getIgnoreMalformed() {
-        return ignoreMalformed;
-    }
-
     public Long getNullValue() {
         return nullValue;
     }
+
+    @Override
+    protected Explicit<Boolean> ignoreMalformed() {
+        return ignoreMalformed;
+    }
 }

server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java

@@ -9,6 +9,7 @@
 package org.opensearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
+import org.opensearch.common.Explicit;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.analysis.IndexAnalyzers;
@@ -71,7 +72,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             m -> toType(m).format,
             getDefaultDateTimeFormatter().pattern()
         );
-        private final Parameter<Boolean> ignoreMalformed;
+        private final Parameter<Explicit<Boolean>> ignoreMalformed;
 
         private static Map<String, Object> parseProperties(String name, Object propertiesObject) {
             if (propertiesObject instanceof Map == false) {
@@ -114,7 +115,12 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, DateFormatter default
             if (defaultDateFormatter != null) {
                 this.format.setValue(defaultDateFormatter.pattern());
             }
-            this.ignoreMalformed = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, defaultIgnoreMalformed);
+            this.ignoreMalformed = Parameter.explicitBoolParam(
+                "ignore_malformed",
+                true,
+                m -> toType(m).ignoreMalformed,
+                defaultIgnoreMalformed
+            );
         }
 
         public Builder(
@@ -136,7 +142,7 @@ public Builder(
                 this.format.setValue(derivedField.getFormat());
             }
             if (derivedField.getIgnoreMalformed()) {
-                this.ignoreMalformed.setValue(derivedField.getIgnoreMalformed());
+                this.ignoreMalformed.setValue(new Explicit<>(derivedField.getIgnoreMalformed(), true));
             }
         }
 
@@ -158,7 +164,7 @@ public DerivedFieldMapper build(BuilderContext context) {
                 derivedField.setFormat(format.getValue());
             }
             if (ignoreMalformed.isConfigured()) {
-                derivedField.setIgnoreMalformed(ignoreMalformed.getValue());
+                derivedField.setIgnoreMalformed(ignoreMalformed.getValue().value());
             }
             FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context, indexAnalyzers);
             Function<Object, IndexableField> fieldFunction = DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(
@@ -193,7 +199,7 @@ public DerivedFieldMapper build(BuilderContext context) {
     private final Script script;
     private final String prefilterField;
     private final Map<String, Object> properties;
-    private final boolean ignoreMalformed;
+    private final Explicit<Boolean> ignoreMalformed;
     private final boolean defaultIgnoreMalformed;
     private final DateFormatter defaultDateFormatter;
     private final String format;
@@ -257,4 +263,8 @@ public Script getScript() {
         return script;
     }
 
+    @Override
+    protected Explicit<Boolean> ignoreMalformed() {
+        return ignoreMalformed;
+    }
 }

server/src/main/java/org/opensearch/index/mapper/FieldMapper.java

@@ -36,13 +36,15 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReader;
+import org.opensearch.common.Explicit;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.AbstractXContentParser;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.NamedAnalyzer;
 import org.opensearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType;
 
@@ -264,15 +266,28 @@ public boolean parsesArrayValue() {
         return false;
     }
 
+    private boolean shouldIgnoreMalformed(IndexSettings is) {
+        if (ignoreMalformed() != null) {
+            return ignoreMalformed().value();
+        }
+
+        if (is == null) {
+            return false;
+        }
+
+        return IGNORE_MALFORMED_SETTING.get(is.getSettings());
+    }
+
     /**
      * Parse the field value using the provided {@link ParseContext}.
      */
     public void parse(ParseContext context) throws IOException {
         try {
             parseCreateField(context);
         } catch (Exception e) {
-            boolean ignore_malformed = false;
-            if (context.indexSettings() != null) ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
+
+            boolean ignoreMalformed = shouldIgnoreMalformed(context.indexSettings());
+
             String valuePreview = "";
             try {
                 XContentParser parser = context.parser();
@@ -283,7 +298,7 @@ public void parse(ParseContext context) throws IOException {
                     valuePreview = complexValue.toString();
                 }
             } catch (Exception innerException) {
-                if (ignore_malformed == false) {
+                if (ignoreMalformed == false) {
                     throw new MapperParsingException(
                         "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
                         e,
@@ -294,7 +309,7 @@ public void parse(ParseContext context) throws IOException {
                 }
             }
 
-            if (ignore_malformed == false) {
+            if (ignoreMalformed == false) {
                 throw new MapperParsingException(
                     "failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
                     e,
@@ -574,6 +589,13 @@ protected static String indexOptionToString(IndexOptions indexOption) {
 
     protected abstract String contentType();
 
+    /**
+     * @return null if a mapper doesn't support setting of `ignore_malformed`
+     */
+    protected Explicit<Boolean> ignoreMalformed() {
+        return null;
+    }
+
     /**
      * Method to create derived source generator for this field mapper, it is illegal to enable the
      * derived source feature and not implement this method for a field mapper