Updating the FieldMappers to insert call addField of DocumentInput

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeaturesFieldMapper.java

@@ -209,6 +209,11 @@ protected void parseCreateField(ParseContext context) {
         throw new AssertionError("parse is implemented directly");
     }
 
+    @Override
+    protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException {
+        throw new AssertionError("parse is implemented directly");
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -451,7 +451,36 @@ protected ScaledFloatFieldMapper clone() {
 
     @Override
     protected void parseCreateField(ParseContext context) throws IOException {
+        Long scaledValue = parseScaledValue(context);
+        if (scaledValue == null) {
+            return;
+        }
+
+        List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
+            fieldType().name(),
+            scaledValue,
+            indexed,
+            hasDocValues,
+            skiplist,
+            stored
+        );
+        context.doc().addAll(fields);
+
+        if (hasDocValues == false && (indexed || stored)) {
+            createFieldNamesField(context);
+        }
+    }
+
+    @Override
+    protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException {
+        Long scaledValue = parseScaledValue(context);
+        if (scaledValue == null) {
+            return;
+        }
+        context.documentInput().addField(fieldType(), scaledValue);
+    }
 
+    private Long parseScaledValue(ParseContext context) throws IOException {
         XContentParser parser = context.parser();
         Object value;
         Number numericValue = null;
@@ -466,7 +495,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
                 numericValue = parse(parser, coerce.value());
             } catch (IllegalArgumentException | JsonParseException e) {
                 if (ignoreMalformed.value()) {
-                    return;
+                    return null;
                 } else {
                     throw e;
                 }
@@ -479,7 +508,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
 
         if (value == null) {
-            return;
+            return null;
         }
 
         if (numericValue == null) {
@@ -489,27 +518,13 @@ protected void parseCreateField(ParseContext context) throws IOException {
         double doubleValue = numericValue.doubleValue();
         if (Double.isFinite(doubleValue) == false) {
             if (ignoreMalformed.value()) {
-                return;
+                return null;
             } else {
                 // since we encode to a long, we have no way to carry NaNs and infinities
                 throw new IllegalArgumentException("[scaled_float] only supports finite values, but got [" + doubleValue + "]");
             }
         }
-        long scaledValue = Math.round(doubleValue * scalingFactor);
-
-        List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
-            fieldType().name(),
-            scaledValue,
-            indexed,
-            hasDocValues,
-            skiplist,
-            stored
-        );
-        context.doc().addAll(fields);
-
-        if (hasDocValues == false && (indexed || stored)) {
-            createFieldNamesField(context);
-        }
+        return Math.round(doubleValue * scalingFactor);
     }
 
     static Double parse(Object value) {

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -489,6 +489,11 @@ protected void parseCreateField(ParseContext context) {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        protected void parseCreateFieldForPluggableFormat(ParseContext context) {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
@@ -525,6 +530,11 @@ protected void parseCreateField(ParseContext context) {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        protected void parseCreateFieldForPluggableFormat(ParseContext context) {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
@@ -650,7 +660,7 @@ public SearchAsYouTypeFieldMapper(
 
     @Override
     protected void parseCreateField(ParseContext context) throws IOException {
-        final String value = context.externalValueSet() ? context.externalValue().toString() : context.parser().textOrNull();
+        final String value = extractValue(context);
         if (value == null) {
             return;
         }
@@ -665,6 +675,19 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
     }
 
+    @Override
+    protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException {
+        final String value = extractValue(context);
+        if (value == null) {
+            return;
+        }
+        context.documentInput().addField(fieldType(), value);
+    }
+
+    private String extractValue(ParseContext context) throws IOException {
+        return context.externalValueSet() ? context.externalValue().toString() : context.parser().textOrNull();
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java

@@ -167,6 +167,27 @@ protected TokenCountFieldMapper(
 
     @Override
     protected void parseCreateField(ParseContext context) throws IOException {
+        final int tokenCount = parseTokenCount(context);
+        if (tokenCount == Integer.MIN_VALUE) {
+            return;
+        }
+
+        context.doc()
+            .addAll(
+                NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store)
+            );
+    }
+
+    @Override
+    protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException {
+        final int tokenCount = parseTokenCount(context);
+        if (tokenCount == Integer.MIN_VALUE) {
+            return;
+        }
+        context.documentInput().addField(fieldType(), tokenCount);
+    }
+
+    private int parseTokenCount(ParseContext context) throws IOException {
         final String value;
         if (context.externalValueSet()) {
             value = context.externalValue().toString();
@@ -175,20 +196,13 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
 
         if (value == null && nullValue == null) {
-            return;
+            return Integer.MIN_VALUE;
         }
 
-        final int tokenCount;
         if (value == null) {
-            tokenCount = nullValue;
-        } else {
-            tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
+            return nullValue;
         }
-
-        context.doc()
-            .addAll(
-                NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store)
-            );
+        return countPositions(analyzer, name(), value, enablePositionIncrements);
     }
 
     /**

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -38,6 +38,8 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.plugins.Plugin;
 
@@ -157,4 +159,23 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
             e.getCause().getMessage()
         );
     }
+
+    public void testParseCreateFieldForPluggableFormat() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
+        RankFeatureFieldMapper fieldMapper = (RankFeatureFieldMapper) mapper.mappers().getMapper("field");
+        assertNotNull(fieldMapper);
+        assertEquals("rank_feature", fieldMapper.typeName());
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatRankFeatureThrows() throws IOException {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping));
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        MapperParsingException e = expectThrows(
+            MapperParsingException.class,
+            () -> mapper.parse(source(b -> b.field("field", 10)), docInput)
+        );
+        assertThat(e.getCause(), instanceOf(UnsupportedOperationException.class));
+    }
 }

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java

@@ -155,4 +155,18 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
             e.getCause().getMessage()
         );
     }
+
+    public void testParseCreateFieldForPluggableFormat() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
+        RankFeaturesFieldMapper fieldMapper = (RankFeaturesFieldMapper) mapper.mappers().getMapper("field");
+        assertNotNull(fieldMapper);
+        assertEquals("rank_features", fieldMapper.typeName());
+    }
+
+    public void testPluggableDataFormatRankFeaturesThrows() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
+        RankFeaturesFieldMapper rfMapper = (RankFeaturesFieldMapper) mapper.mappers().getMapper("field");
+        expectThrows(AssertionError.class, () -> rfMapper.parseCreateFieldForPluggableFormat(null));
+    }
+
 }

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -44,7 +44,9 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.store.Directory;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
@@ -524,4 +526,83 @@ public void testSkiplistParameter() throws IOException {
         );
         assertThat(e.getMessage(), containsString("Failed to parse value [invalid] as only [true] or [false] are allowed"));
     }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatScaledFloatValue() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject())
+        );
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(source(b -> b.field("field", 3.14)), docInput);
+
+        boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field"));
+        assertTrue("Expected scaled float field to be captured", found);
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggableDataFormatScaledFloatNullSkipped() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+        DocumentMapper mapper = createDocumentMapper(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject())
+        );
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        mapper.parse(source(b -> b.nullField("field")), docInput);
+
+        boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field"));
+        assertFalse("Expected no field entry for null value", found);
+    }
+
+    @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG)
+    public void testPluggablePathEquivalenceWithLucenePath() throws Exception {
+        Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build();
+
+        // Scenario 1: scaled float value
+        assertScaledFloatLuceneAndPluggablePathsEquivalent(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject()),
+            b -> b.field("field", 3.14),
+            "field",
+            true
+        );
+
+        // Scenario 2: null value — no field produced
+        assertScaledFloatLuceneAndPluggablePathsEquivalent(
+            pluggableSettings,
+            mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject()),
+            b -> b.nullField("field"),
+            "field",
+            false
+        );
+    }
+
+    private void assertScaledFloatLuceneAndPluggablePathsEquivalent(
+        Settings pluggableSettings,
+        XContentBuilder mappingBuilder,
+        CheckedConsumer<XContentBuilder, IOException> sourceBuilder,
+        String fieldName,
+        boolean expectField
+    ) throws IOException {
+        // Lucene path
+        DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder);
+        ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder));
+        IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName);
+
+        // Pluggable path
+        DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder);
+        CapturingDocumentInput docInput = new CapturingDocumentInput();
+        pluggableMapper.parse(source(sourceBuilder), docInput);
+
+        boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName));
+
+        if (!expectField) {
+            assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length);
+            assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField);
+        } else {
+            assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0);
+            assertTrue("Pluggable path should capture field '" + fieldName + "'", pluggableHasField);
+        }
+    }
 }