From 2e327859a18b8c4af61a65b5020be0cdc5eb4fc6 Mon Sep 17 00:00:00 2001 From: Sagar Darji Date: Mon, 23 Mar 2026 15:09:23 +0530 Subject: [PATCH 1/3] Following changes are part of this PR: 1. Introducing FeatureFlag and Index Setting for pluggable dataformat feature. 2. Updating the FieldMappers to insert fields in DocumentInput for Multi Format Engine Signed-off-by: Sagar Darji # Conflicts: # CHANGELOG.md --- .../index/mapper/ScaledFloatFieldMapper.java | 26 ++-- .../mapper/SearchAsYouTypeFieldMapper.java | 18 ++- .../index/mapper/TokenCountFieldMapper.java | 12 +- .../mapper/ScaledFloatFieldMapperTests.java | 29 ++++ .../SearchAsYouTypeFieldMapperTests.java | 32 +++++ .../mapper/TokenCountFieldMapperTests.java | 43 ++++++ .../join/mapper/ParentIdFieldMapper.java | 10 +- .../join/mapper/ParentJoinFieldMapper.java | 11 +- .../percolator/QueryBuilderStoreTests.java | 6 + .../ICUCollationKeywordFieldMapper.java | 21 +-- .../index/mapper/size/SizeFieldMapper.java | 6 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../common/settings/IndexScopedSettings.java | 1 + .../opensearch/common/util/FeatureFlags.java | 12 ++ .../org/opensearch/index/IndexSettings.java | 7 + .../index/mapper/BinaryFieldMapper.java | 32 +++-- .../index/mapper/BooleanFieldMapper.java | 22 +-- .../index/mapper/DateFieldMapper.java | 30 +++-- .../index/mapper/DocumentMapper.java | 5 + .../index/mapper/DocumentParser.java | 20 ++- .../opensearch/index/mapper/FieldMapper.java | 12 +- .../index/mapper/IpFieldMapper.java | 28 ++-- .../index/mapper/KeywordFieldMapper.java | 26 ++-- .../org/opensearch/index/mapper/Mapper.java | 15 +++ .../index/mapper/NumberFieldMapper.java | 10 +- .../opensearch/index/mapper/ParseContext.java | 29 ++++ .../index/mapper/ParsedDocument.java | 21 +++ .../index/mapper/RangeFieldMapper.java | 10 +- .../index/mapper/TextFieldMapper.java | 4 +- .../index/mapper/BinaryFieldMapperTests.java | 32 +++++ .../index/mapper/BooleanFieldMapperTests.java | 43 ++++++ .../index/mapper/DateFieldMapperTests.java | 30 +++++ .../index/mapper/DocumentMapperTests.java | 89 +++++++++++++ .../index/mapper/DocumentParserTests.java | 87 ++++++++++++ .../index/mapper/IpFieldMapperTests.java | 30 +++++ .../index/mapper/KeywordFieldMapperTests.java | 125 ++++++++++++++++++ .../opensearch/index/mapper/MapperTests.java | 15 +++ .../index/mapper/NumberFieldMapperTests.java | 50 +++++++ .../index/mapper/TextFieldMapperTests.java | 26 ++++ .../index/mapper/MapperServiceTestCase.java | 75 +++++++++++ 40 files changed, 994 insertions(+), 109 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index 66bde201805ee..fa5f9772f735f 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -497,18 +497,22 @@ protected void parseCreateField(ParseContext context) throws IOException { } long scaledValue = Math.round(doubleValue * scalingFactor); - List fields = NumberFieldMapper.NumberType.LONG.createFields( - fieldType().name(), - scaledValue, - indexed, - hasDocValues, - skiplist, - stored - ); - context.doc().addAll(fields); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), scaledValue); + } else { + List fields = NumberFieldMapper.NumberType.LONG.createFields( + fieldType().name(), + scaledValue, + indexed, + hasDocValues, + skiplist, + stored + ); + context.doc().addAll(fields); - if (hasDocValues == false && (indexed || stored)) { - createFieldNamesField(context); + if (hasDocValues == false && (indexed || stored)) { + createFieldNamesField(context); + } } } diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java index f08815ebbbd1e..7b973fd50b10c 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -655,13 +655,17 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); - for (ShingleFieldMapper subFieldMapper : shingleFields) { - context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); - } - context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); - if (fieldType().fieldType.omitNorms()) { - createFieldNamesField(context); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), value); + } else { + context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); + for (ShingleFieldMapper subFieldMapper : shingleFields) { + context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); + } + context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); + if (fieldType().fieldType.omitNorms()) { + createFieldNamesField(context); + } } } diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java index 929a9890a9ec9..f9af31c4f6fde 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java @@ -185,10 +185,14 @@ protected void parseCreateField(ParseContext context) throws IOException { tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements); } - context.doc() - .addAll( - NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store) - ); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), tokenCount); + } else { + context.doc() + .addAll( + NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store) + ); + } } /** diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java index 08354f786bac8..d631bb142a4af 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -45,6 +45,7 @@ import org.apache.lucene.store.Directory; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -524,4 +525,32 @@ public void testSkiplistParameter() throws IOException { ); assertThat(e.getMessage(), containsString("Failed to parse value [invalid] as only [true] or [false] are allowed")); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatScaledFloatValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", 3.14)), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected scaled float field to be captured", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatScaledFloatNullSkipped() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no field entry for null value", found); + } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index d218f89ff154d..d5b887d70de34 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -52,6 +52,8 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.TermQuery; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -769,4 +771,34 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp assertThat(mapper, instanceOf(PrefixFieldMapper.class)); return (PrefixFieldMapper) mapper; } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatSearchAsYouTypeValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "hello world")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("hello world")); + assertTrue("Expected search_as_you_type value", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatSearchAsYouTypeNullSkipped() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no field entry for null value", found); + } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java index dd0f7485c6e4f..7e97421bc9228 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java @@ -42,6 +42,8 @@ import org.apache.lucene.tests.analysis.CannedTokenStream; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.tests.analysis.Token; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalyzerScope; @@ -219,4 +221,45 @@ private SourceToParse createDocument(String fieldValue) throws Exception { private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse request) { return mapper.parse(request).docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed")); } + + private DocumentMapper createIndexWithTokenCountFieldPluggableDataFormat() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + return createDocumentMapper(pluggableSettings, mapping(b -> { + b.startObject("test"); + { + b.field("type", "text"); + b.startObject("fields"); + { + b.startObject("tc"); + { + b.field("type", "token_count"); + b.field("analyzer", "standard"); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatTokenCountValue() throws Exception { + DocumentMapper mapper = createIndexWithTokenCountFieldPluggableDataFormat(); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(createDocument("three tokens string"), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc") && e.getValue().equals(3)); + assertTrue("Expected token count of 3 for field test.tc", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatTokenCountNullSkipped() throws Exception { + DocumentMapper mapper = createIndexWithTokenCountFieldPluggableDataFormat(); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(createDocument(null), docInput); + + boolean hasTokenCountField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc")); + assertFalse("Expected no token count field for null value", hasTokenCountField); + } } diff --git a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java index 29a02a5bc8032..57bf6eb9ca504 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java @@ -203,9 +203,13 @@ protected void parseCreateField(ParseContext context) throws IOException { } String refId = (String) context.externalValue(); BytesRef binaryValue = new BytesRef(refId); - Field field = new Field(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), binaryValue); + } else { + Field field = new Field(fieldType().name(), binaryValue, fieldType); + context.doc().add(field); + context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + } } @Override diff --git a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java index a229d050f3b1d..36093d5f357e9 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java @@ -430,9 +430,14 @@ public void parse(ParseContext context) throws IOException { } BytesRef binaryValue = new BytesRef(name); - Field field = new Field(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), binaryValue); + } else { + Field field = new Field(fieldType().name(), binaryValue, fieldType); + context.doc().add(field); + context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + } + context.path().remove(); } diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java index 9a708a823507f..28e38ae5e3e6e 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/QueryBuilderStoreTests.java @@ -100,6 +100,12 @@ public void testStoringQueryBuilders() throws IOException { ParseContext parseContext = mock(ParseContext.class); ParseContext.Document document = new ParseContext.Document(); when(parseContext.doc()).thenReturn(document); + when(parseContext.indexSettings()).thenReturn( + new org.opensearch.index.IndexSettings( + IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(), + settings + ) + ); PercolatorFieldMapper.createQueryBuilderField(version, fieldMapper, queryBuilders[i], parseContext); indexWriter.addDocument(document); } diff --git a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java index 4408ef51c3a19..b37dfa6c40dac 100644 --- a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -812,15 +812,20 @@ protected void parseCreateField(ParseContext context) throws IOException { RawCollationKey key = collator.getRawCollationKey(value, null); final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size); - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new Field(mappedFieldType.name(), binaryValue, fieldType); - context.doc().add(field); - } + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), binaryValue); + } else { - if (fieldType().hasDocValues()) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - createFieldNamesField(context); + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + Field field = new Field(mappedFieldType.name(), binaryValue, fieldType); + context.doc().add(field); + } + + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + createFieldNamesField(context); + } } } diff --git a/plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java index dc966a3dfc50f..f16c9a40390d6 100644 --- a/plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java @@ -99,7 +99,11 @@ public void postParse(ParseContext context) throws IOException { return; } final int value = context.sourceToParse().source().length(); - context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), value); + } else { + context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true)); + } } @Override diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 6e81243e3b9b6..8ebf3bf207bb1 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,6 +37,7 @@ protected FeatureFlagSettings( FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING, - FeatureFlags.STREAM_TRANSPORT_SETTING + FeatureFlags.STREAM_TRANSPORT_SETTING, + FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 3494f5557d7b3..a3a49bb2fd9e9 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -303,6 +303,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Setting for derived source feature IndexSettings.INDEX_DERIVED_SOURCE_SETTING, IndexSettings.INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING, + IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING, // Writable warm / tiering settings - always registered so nodes can parse // index metadata even when the feature flag is disabled diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 904f3594b68de..2a53dc05f6559 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -74,6 +74,17 @@ public class FeatureFlags { Property.NodeScope ); + /** + * Gates the functionality of pluggable dataformat feature. + */ + public static final String PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG = FEATURE_FLAG_PREFIX + "pluggable.dataformat.enabled"; + + public static final Setting PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING = Setting.boolSetting( + PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, + false, + Property.NodeScope + ); + public static final Setting CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_FLAG, false, @@ -141,6 +152,7 @@ static class FeatureFlagsImpl { put(TERM_VERSION_PRECOMMIT_ENABLE_SETTING, TERM_VERSION_PRECOMMIT_ENABLE_SETTING.getDefault(Settings.EMPTY)); put(STREAM_TRANSPORT_SETTING, STREAM_TRANSPORT_SETTING.getDefault(Settings.EMPTY)); put(CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_SETTING, CONTEXT_AWARE_MIGRATION_EXPERIMENTAL_SETTING.getDefault(Settings.EMPTY)); + put(PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING, PLUGGABLE_DATAFORMAT_EXPERIMENTAL_SETTING.getDefault(Settings.EMPTY)); } }; diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 2dc33b3017572..4f9c2c9e82038 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -917,6 +917,13 @@ public static IndexMergePolicy fromString(String text) { Property.Dynamic ); + public static final Setting PLUGGABLE_DATAFORMAT_ENABLED_SETTING = Setting.boolSetting( + "index.pluggable.dataformat.enabled", + false, + Property.IndexScope, + Property.Final + ); + private final Index index; private final Version version; private final Logger logger; diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index 040491f775357..cdd217e6d6eec 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -205,23 +205,27 @@ protected void parseCreateField(ParseContext context) throws IOException { if (value == null) { return; } - if (stored) { - context.doc().add(new StoredField(fieldType().name(), value)); - } + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), value); + } else { + if (stored) { + context.doc().add(new StoredField(fieldType().name(), value)); + } - if (hasDocValues) { - CustomBinaryDocValuesField field = (CustomBinaryDocValuesField) context.doc().getByKey(fieldType().name()); - if (field == null) { - field = new CustomBinaryDocValuesField(fieldType().name(), value); - context.doc().addWithKey(fieldType().name(), field); + if (hasDocValues) { + CustomBinaryDocValuesField field = (CustomBinaryDocValuesField) context.doc().getByKey(fieldType().name()); + if (field == null) { + field = new CustomBinaryDocValuesField(fieldType().name(), value); + context.doc().addWithKey(fieldType().name(), field); + } else { + field.add(value); + } } else { - field.add(value); + // Only add an entry to the field names field if the field is stored + // but has no doc values so exists query will work on a field with + // no doc values + createFieldNamesField(context); } - } else { - // Only add an entry to the field names field if the field is stored - // but has no doc values so exists query will work on a field with - // no doc values - createFieldNamesField(context); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index f3a027e3375e9..be0c53c8fd4c4 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -389,16 +389,20 @@ protected void parseCreateField(ParseContext context) throws IOException { if (value == null) { return; } - if (indexed) { - context.doc().add(new Field(fieldType().name(), value ? "T" : "F", Defaults.FIELD_TYPE)); - } - if (stored) { - context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); - } - if (hasDocValues) { - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), value); } else { - createFieldNamesField(context); + if (indexed) { + context.doc().add(new Field(fieldType().name(), value ? "T" : "F", Defaults.FIELD_TYPE)); + } + if (stored) { + context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); + } + if (hasDocValues) { + context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + } else { + createFieldNamesField(context); + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 25f22838b7bb8..bd33263594a19 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -834,20 +834,24 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - if (indexed) { - context.doc().add(new LongPoint(fieldType().name(), timestamp)); - } - if (hasDocValues) { - if (skiplist || isSkiplistDefaultEnabled(context.indexSettings().getIndexSortConfig(), fieldType().name())) { - context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp)); - } else { - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), timestamp); + } else { + if (indexed) { + context.doc().add(new LongPoint(fieldType().name(), timestamp)); + } + if (hasDocValues) { + if (skiplist || isSkiplistDefaultEnabled(context.indexSettings().getIndexSortConfig(), fieldType().name())) { + context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp)); + } else { + context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); + } + } else if (store || indexed) { + createFieldNamesField(context); + } + if (store) { + context.doc().add(new StoredField(fieldType().name(), timestamp)); } - } else if (store || indexed) { - createFieldNamesField(context); - } - if (store) { - context.doc().add(new StoredField(fieldType().name(), timestamp)); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java index cb7e08f062d6d..9a528544ca441 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapper.java @@ -51,6 +51,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.IndexSortConfig; import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.index.mapper.MetadataFieldMapper.TypeParser; import org.opensearch.index.query.NestedQueryBuilder; @@ -253,6 +254,10 @@ public ParsedDocument parse(SourceToParse source) throws MapperParsingException return documentParser.parseDocument(source, mapping.metadataMappers); } + public ParsedDocument parse(SourceToParse source, DocumentInput documentInput) throws MapperParsingException { + return documentParser.parseDocument(source, mapping.metadataMappers, documentInput); + } + public ParsedDocument createDeleteTombstoneDoc(String index, String id) throws MapperParsingException { final SourceToParse emptySource = new SourceToParse(index, id, new BytesArray("{}"), MediaTypeRegistry.JSON); return documentParser.parseDocument(emptySource, deleteTombstoneMetadataFieldMappers).toTombstone(); diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index ba1bf1d189c7f..02ff66b51af7b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -49,6 +49,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.DynamicTemplate.XContentFieldType; import org.opensearch.script.ContextAwareGroupingScript; @@ -82,6 +83,11 @@ final class DocumentParser { } ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadataFieldsMappers) throws MapperParsingException { + return parseDocument(source, metadataFieldsMappers, null); + } + + ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadataFieldsMappers, DocumentInput documentInput) + throws MapperParsingException { final Mapping mapping = docMapper.mapping(); final ParseContext.InternalParseContext context; final MediaType mediaType = source.getMediaType(); @@ -94,7 +100,7 @@ ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadat mediaType ) ) { - context = new ParseContext.InternalParseContext(indexSettings, docMapperParser, docMapper, source, parser); + context = new ParseContext.InternalParseContext(indexSettings, docMapperParser, docMapper, source, parser, documentInput); validateStart(parser); internalParseDocument(mapping, metadataFieldsMappers, context, parser); validateEnd(parser); @@ -108,7 +114,7 @@ ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadat context.postParse(); - return parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers())); + return parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers()), documentInput); } private static boolean containsDisabledObjectMapper(ObjectMapper objectMapper, String[] subfields) { @@ -182,7 +188,12 @@ private static boolean isEmptyDoc(Mapping mapping, XContentParser parser) throws return false; } - private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.InternalParseContext context, Mapping update) { + private static ParsedDocument parsedDocument( + SourceToParse source, + ParseContext.InternalParseContext context, + Mapping update, + DocumentInput documentInput + ) { return new ParsedDocument( context.version(), context.seqID(), @@ -191,7 +202,8 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext. context.docs(), context.sourceToParse().source(), context.sourceToParse().getMediaType(), - update + update, + documentInput ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index 5ba68e6d7cb80..ab59867b68087 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -60,6 +60,7 @@ import java.util.Objects; import java.util.Spliterators; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.StreamSupport; /** @@ -217,6 +218,7 @@ public T meta(Map meta) { protected MultiFields multiFields; protected CopyTo copyTo; protected DerivedFieldGenerator derivedFieldGenerator; + private final AtomicBoolean isPluggableDataFormatFeatureEnabled = new AtomicBoolean(false); protected FieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { super(simpleName); @@ -366,11 +368,19 @@ protected final void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) { for (String fieldName : FieldNamesFieldMapper.extractFieldNames(fieldType().name())) { - context.doc().add(new Field(FieldNamesFieldMapper.NAME, fieldName, FieldNamesFieldMapper.Defaults.FIELD_TYPE)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldNamesFieldType, fieldName); + } else { + context.doc().add(new Field(FieldNamesFieldMapper.NAME, fieldName, FieldNamesFieldMapper.Defaults.FIELD_TYPE)); + } } } } + protected final boolean isPluggableDataFormatFeatureEnabled(ParseContext parseContext) { + return isPluggableDataFormatEnabled(parseContext.indexSettings().getSettings()); + } + @Override public Iterator iterator() { return multiFields.iterator(); diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 992007329a135..903391550e802 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -638,18 +638,22 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - if (indexed && hasDocValues) { - context.doc().add(new InetAddressField(fieldType().name(), address)); - } else if (indexed) { - context.doc().add(new InetAddressPoint(fieldType().name(), address)); - } else if (hasDocValues) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); - } - if ((stored || indexed) && hasDocValues == false) { - createFieldNamesField(context); - } - if (stored) { - context.doc().add(new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), address); + } else { + if (indexed && hasDocValues) { + context.doc().add(new InetAddressField(fieldType().name(), address)); + } else if (indexed) { + context.doc().add(new InetAddressPoint(fieldType().name(), address)); + } else if (hasDocValues) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + } + if ((stored || indexed) && hasDocValues == false) { + createFieldNamesField(context); + } + if (stored) { + context.doc().add(new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 3271f60a466ee..c4b0f798eb7bd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -881,19 +881,23 @@ protected void parseCreateField(ParseContext context) throws IOException { value = normalizeValue(normalizer, name(), value); } - // convert to utf8 only once before feeding postings/dv/stored fields - final BytesRef binaryValue = new BytesRef(value); - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new KeywordField(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { - createFieldNamesField(context); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), value); + } else { + // convert to utf8 only once before feeding postings/dv/stored fields + final BytesRef binaryValue = new BytesRef(value); + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + Field field = new KeywordField(fieldType().name(), binaryValue, fieldType); + context.doc().add(field); + + if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + createFieldNamesField(context); + } } - } - if (fieldType().hasDocValues()) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java index 3b9024162656f..939f7de12e912 100644 --- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java @@ -39,6 +39,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.analysis.IndexAnalyzers; @@ -52,6 +53,8 @@ import java.util.function.Function; import java.util.function.Supplier; +import static org.opensearch.index.IndexSettings.PLUGGABLE_DATAFORMAT_ENABLED_SETTING; + /** * The foundation OpenSearch mapper * @@ -303,6 +306,18 @@ protected static boolean hasIndexCreated(Settings settings) { return settings.hasValue(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey()); } + /** + * Checks if the optimised index feature is enabled for the given settings. + * Requires both the {@link FeatureFlags#PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG} feature flag + * + * @param settings the index settings to check + * @return {@code true} if the pluggable dataformat feature flag and the optimised index setting are both enabled + */ + public static boolean isPluggableDataFormatEnabled(Settings settings) { + return FeatureFlags.isEnabled(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + && PLUGGABLE_DATAFORMAT_ENABLED_SETTING.get(settings); + } + /** * Method to determine, if it is possible to derive source for this field using field mapping parameters */ diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 3a7f942ef4525..ae67161c6db7b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -2167,10 +2167,14 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, skiplist, stored)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), numericValue); + } else { + context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, skiplist, stored)); - if (hasDocValues == false && (stored || indexed)) { - createFieldNamesField(context); + if (hasDocValues == false && (stored || indexed)) { + createFieldNamesField(context); + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/ParseContext.java b/server/src/main/java/org/opensearch/index/mapper/ParseContext.java index aaad2d1922d7a..d37c147c90e06 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParseContext.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParseContext.java @@ -36,9 +36,11 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchParseException; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DocumentInput; import java.util.ArrayList; import java.util.Collection; @@ -263,6 +265,11 @@ public Document doc() { return in.doc(); } + @Override + public DocumentInput documentInput() { + return in.documentInput(); + } + @Override protected void addDoc(Document doc) { in.addDoc(doc); @@ -414,12 +421,25 @@ public static class InternalParseContext extends ParseContext { private final Set ignoredFields = new HashSet<>(); + private DocumentInput documentInput; + public InternalParseContext( IndexSettings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, SourceToParse source, XContentParser parser + ) { + this(indexSettings, docMapperParser, docMapper, source, parser, null); + } + + public InternalParseContext( + IndexSettings indexSettings, + DocumentMapperParser docMapperParser, + DocumentMapper docMapper, + SourceToParse source, + XContentParser parser, + DocumentInput documentInput ) { this.indexSettings = indexSettings; this.docMapper = docMapper; @@ -427,6 +447,7 @@ public InternalParseContext( this.path = new ContentPath(0); this.parser = parser; this.document = new Document(); + this.documentInput = documentInput; this.documents = new ArrayList<>(); this.documents.add(document); this.version = null; @@ -479,6 +500,11 @@ public Document doc() { return this.document; } + @Override + public DocumentInput documentInput() { + return this.documentInput; + } + @Override protected void addDoc(Document doc) { numNestedDocs++; @@ -749,6 +775,9 @@ public boolean isWithinMultiFields() { public abstract Document doc(); + @ExperimentalApi + public abstract DocumentInput documentInput(); + protected abstract void addDoc(Document doc); public abstract RootObjectMapper root(); diff --git a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java index 16e38980f8600..a29e88a51f714 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java @@ -37,6 +37,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.index.mapper.ParseContext.Document; @@ -63,6 +64,7 @@ public class ParsedDocument { private MediaType mediaType; private Mapping dynamicMappingsUpdate; + private DocumentInput documentInput; public ParsedDocument( Field version, @@ -73,6 +75,20 @@ public ParsedDocument( BytesReference source, MediaType mediaType, Mapping dynamicMappingsUpdate + ) { + this(version, seqID, id, routing, documents, source, mediaType, dynamicMappingsUpdate, null); + } + + public ParsedDocument( + Field version, + SeqNoFieldMapper.SequenceIDFields seqID, + String id, + String routing, + List documents, + BytesReference source, + MediaType mediaType, + Mapping dynamicMappingsUpdate, + DocumentInput documentInput ) { this.version = version; this.seqID = seqID; @@ -82,6 +98,11 @@ public ParsedDocument( this.source = source; this.dynamicMappingsUpdate = dynamicMappingsUpdate; this.mediaType = mediaType; + this.documentInput = documentInput; + } + + public DocumentInput getDocumentInput() { + return documentInput; } public String id() { diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index 71e0353b32db7..1a1fb2c43454c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -474,10 +474,14 @@ protected void parseCreateField(ParseContext context) throws IOException { if (range == null) { return; } - context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store)); + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), range); + } else { + context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store)); - if (hasDocValues == false && (index || store)) { - createFieldNamesField(context); + if (hasDocValues == false && (index || store)) { + createFieldNamesField(context); + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java index 50b30e65e315a..2cdc5313e603b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java @@ -1040,7 +1040,9 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + if (isPluggableDataFormatFeatureEnabled(context)) { + context.documentInput().addField(fieldType(), value); + } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { Field field = new Field(fieldType().name(), value, fieldType); context.doc().add(field); if (fieldType.omitNorms()) { diff --git a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java index 87b5ad3434944..cdb49d4627303 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java @@ -34,6 +34,8 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.compress.CompressorRegistry; @@ -135,4 +137,34 @@ public void testStoredValue() throws IOException { assertEquals(new BytesArray(value), originalValue); } } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatBinaryValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + byte[] testValue = new byte[] { 1, 2, 3 }; + String base64Value = java.util.Base64.getEncoder().encodeToString(testValue); + mapper.parse(source(b -> b.field("field", base64Value)), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected binary field to be captured", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatBinaryNullSkipped() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no binary field to be captured for null value", found); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 894e76d0ea442..6f53da7299ffc 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -47,6 +47,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Booleans; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -314,4 +316,45 @@ private org.apache.lucene.document.Document createDocument(String value, boolean } return doc; } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatBooleanTrue() throws IOException { + Settings settings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(settings, fieldMapping(this::minimalMapping)); + + CapturingDocumentInput capturingDocInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", true)), capturingDocInput); + + assertTrue( + capturingDocInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && Boolean.TRUE.equals(e.getValue())) + ); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatBooleanFalse() throws IOException { + Settings settings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(settings, fieldMapping(this::minimalMapping)); + + CapturingDocumentInput capturingDocInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", false)), capturingDocInput); + + assertTrue( + capturingDocInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && Boolean.FALSE.equals(e.getValue())) + ); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatBooleanNullSkipped() throws IOException { + Settings settings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(settings, fieldMapping(this::minimalMapping)); + + CapturingDocumentInput capturingDocInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), capturingDocInput); + + assertTrue(capturingDocInput.getCapturedFields().stream().noneMatch(e -> e.getKey().name().equals("field"))); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java index e03cbfdd5bec4..8f714e6b30e39 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java @@ -936,4 +936,34 @@ public void testSkipListIntegrationFieldBehaviorConsistency() throws IOException ); assertEquals("Expected timestamp should match", expectedTimestamp, fieldsEnabled[0].numericValue().longValue()); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatDateValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "date").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "2025-02-18T06:00:00.000Z")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals(TEST_TIMESTAMP)); + assertTrue("Expected timestamp for date field", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatDateNullSkipped() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "date").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasDateField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasDateField); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentMapperTests.java index fa6ef72552faf..0e410c0d791c5 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentMapperTests.java @@ -40,6 +40,7 @@ import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.MapperService.MergeReason; import java.io.IOException; @@ -54,6 +55,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; public class DocumentMapperTests extends MapperServiceTestCase { @@ -291,4 +293,91 @@ public void testMergeMetaForIndexTemplate() throws IOException { expected = Map.of("field", "value", "object", Map.of("field1", "value1", "field2", "new_value", "field3", "value3")); assertThat(mergedMapper.meta(), equalTo(expected)); } + + public void testParseWithoutDocumentInputReturnsNullDocumentInput() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "text").endObject())); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value"))); + + assertThat(doc.getDocumentInput(), nullValue()); + } + + public void testParseWithDocumentInputPropagatesInput() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "text").endObject())); + DocumentInput> mockInput = new MockDocumentInput(); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value")), mockInput); + + assertThat(doc.getDocumentInput(), sameInstance(mockInput)); + } + + public void testParseWithNullDocumentInputExplicitly() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "text").endObject())); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value")), null); + + assertThat(doc.getDocumentInput(), nullValue()); + } + + public void testParseWithDocumentInputMultipleFields() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("name").field("type", "text").endObject(); + b.startObject("age").field("type", "integer").endObject(); + })); + DocumentInput> mockInput = new MockDocumentInput(); + + ParsedDocument doc = mapper.parse(source(b -> { + b.field("name", "test"); + b.field("age", 25); + }), mockInput); + + assertThat(doc.getDocumentInput(), sameInstance(mockInput)); + assertThat(doc.rootDoc().getField("name"), notNullValue()); + assertThat(doc.rootDoc().getField("age"), notNullValue()); + } + + public void testParseWithDocumentInputNestedObject() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("obj"); + { + b.startObject("properties"); + { + b.startObject("field").field("type", "text").endObject(); + } + b.endObject(); + } + b.endObject(); + })); + DocumentInput> mockInput = new MockDocumentInput(); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("obj"); + b.field("field", "value"); + b.endObject(); + }), mockInput); + + assertThat(doc.getDocumentInput(), sameInstance(mockInput)); + } + + private static class MockDocumentInput implements DocumentInput> { + private final Map fields = new HashMap<>(); + + @Override + public Map getFinalInput() { + return Collections.unmodifiableMap(fields); + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + fields.put(fieldType != null ? fieldType.name() : "field_" + fields.size(), value); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) { + fields.put(rowIdFieldName, rowId); + } + + @Override + public void close() {} + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index b99320410b99e..d94d81c59f81c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.ParseContext.Document; import org.opensearch.plugins.Plugin; @@ -52,7 +53,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static java.util.Collections.singletonList; import static org.opensearch.test.StreamsUtils.copyToBytesFromClasspath; @@ -62,6 +65,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; public class DocumentParserTests extends MapperServiceTestCase { @@ -3574,4 +3579,86 @@ public void testGeoPointArrayWithMultipleCopyTo() throws Exception { assertNotNull(copy2Fields); assertTrue(copy2Fields.length > 0); } + + public void testParseDocumentWithDocumentInputPropagated() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "text").endObject())); + DocumentInput> mockInput = new TestDocumentInput(); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value")), mockInput); + + assertThat(doc.getDocumentInput(), sameInstance(mockInput)); + assertNotNull(doc.rootDoc().getField("field")); + } + + public void testParseDocumentWithNullDocumentInput() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "text").endObject())); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value")), null); + + assertThat(doc.getDocumentInput(), nullValue()); + } + + public void testParseDocumentWithoutDocumentInputDefaultsToNull() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "text").endObject())); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value"))); + + assertThat(doc.getDocumentInput(), nullValue()); + } + + public void testParseDocumentWithDocumentInputAndDynamicMapping() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + DocumentInput> mockInput = new TestDocumentInput(); + + ParsedDocument doc = mapper.parse(source(b -> b.field("dynamic_field", "value")), mockInput); + + assertThat(doc.getDocumentInput(), sameInstance(mockInput)); + assertNotNull(doc.dynamicMappingsUpdate()); + } + + public void testParseDocumentWithDocumentInputAndNestedFields() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("obj"); + { + b.startObject("properties"); + { + b.startObject("inner").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + })); + DocumentInput> mockInput = new TestDocumentInput(); + + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("obj"); + b.field("inner", "test"); + b.endObject(); + }), mockInput); + + assertThat(doc.getDocumentInput(), sameInstance(mockInput)); + assertNotNull(doc.rootDoc().getField("obj.inner")); + } + + private static class TestDocumentInput implements DocumentInput> { + private final Map fields = new HashMap<>(); + + @Override + public Map getFinalInput() { + return Collections.unmodifiableMap(fields); + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + fields.put(fieldType != null ? fieldType.name() : "field_" + fields.size(), value); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) { + fields.put(rowIdFieldName, rowId); + } + + @Override + public void close() {} + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java index 54399a31e4456..1e1bc541ef3cc 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java @@ -47,6 +47,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.termvectors.TermVectorsService; @@ -317,4 +319,32 @@ private Document createDocument(String value, boolean hasDocValues) { } return doc; } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIpValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "ip").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field(FIELD_NAME, "192.168.1.1")), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(FIELD_NAME)); + assertTrue("Expected ip field to be captured", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIpNullSkipped() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "ip").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField(FIELD_NAME)), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(FIELD_NAME)); + assertFalse("Expected no ip field to be captured for null value", found); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java index 3a623f46101de..30f4df625b371 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java @@ -51,6 +51,8 @@ import org.apache.lucene.tests.analysis.MockLowerCaseFilter; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; @@ -582,4 +584,127 @@ private Document createDocument(KeywordFieldMapper mapper, String value, boolean } return doc; } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatDefaultKeyword() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "test_value")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("test_value")); + assertTrue("Expected keyword field captured with value 'test_value'", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueConfigured() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("null_value", "default_val").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("default_val")); + assertTrue("Expected keyword field captured with null_value 'default_val'", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIgnoreAbove() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 5).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "opensearch")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field when value exceeds ignore_above", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIgnoreAboveWithinLimit() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 5).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "elk")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("elk")); + assertTrue("Expected keyword field captured with value 'elk' within ignore_above limit", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatWithNormalizer() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("normalizer", "lowercase").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "AbC")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("abc")); + assertTrue("Expected keyword field captured with normalized value 'abc'", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatWithExternalValue() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, mapping(b -> { + b.startObject("text_field"); + b.field("type", "text"); + b.startObject("fields"); + b.startObject("kw").field("type", "keyword").endObject(); + b.endObject(); + b.endObject(); + })); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("text_field", "external_keyword")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("text_field.kw") && e.getValue().equals("external_keyword")); + assertTrue("Expected keyword sub-field captured with external value 'external_keyword'", found); + } + + public void testDefaultsDoNotUseDocumentInput() throws Exception { + XContentBuilder mapping = fieldMapping(this::minimalMapping); + DocumentMapper mapper = createDocumentMapper(mapping); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1234"))); + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + assertEquals(new BytesRef("1234"), fields[0].binaryValue()); + assertEquals(new BytesRef("1234"), fields[1].binaryValue()); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/MapperTests.java b/server/src/test/java/org/opensearch/index/mapper/MapperTests.java index 2b7d83e2ec04b..05327643dc86f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MapperTests.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.test.OpenSearchTestCase; public class MapperTests extends OpenSearchTestCase { @@ -52,4 +53,18 @@ public void testBuilderContextWithIndexSettingsAsNull() { expectThrows(NullPointerException.class, () -> new Mapper.BuilderContext(null, new ContentPath(1))); } + public void testIsOptimisedIndexEnabledReturnsFalseByDefault() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); + assertFalse(Mapper.isPluggableDataFormatEnabled(settings)); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testIsOptimisedIndexEnabledReturnsTrueWhenFlagEnabled() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.pluggable.dataformat.enabled", true) + .build(); + assertTrue(Mapper.isPluggableDataFormatEnabled(settings)); + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java index 7f2efa0ba7fdb..974ec52238f6a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java @@ -50,6 +50,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.NumericUtils; import org.opensearch.common.Numbers; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -739,4 +741,52 @@ public void testNegativeNumberHandling() { decoded = IntPoint.decodeDimension(encoded, 0); assertEquals(-101, decoded); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIntegerValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "integer").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field(FIELD_NAME, 42)), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(FIELD_NAME) && e.getValue().equals(42)); + assertTrue("Expected integer value 42", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatLongValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "long").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field(FIELD_NAME, 123456789L)), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(FIELD_NAME) && e.getValue().equals(123456789L)); + assertTrue("Expected long value 123456789", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatDoubleValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "double").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field(FIELD_NAME, 3.14)), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(FIELD_NAME) && e.getValue().equals(3.14)); + assertTrue("Expected double value 3.14", found); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java index 8cc1c281d67e9..bf3f7d7cd73e2 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java @@ -74,6 +74,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -105,6 +106,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -1173,4 +1175,28 @@ private Document createDocument(String name, String value, boolean forKeyword, b } return doc; } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatTextValue() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(b -> b.field("type", "text"))); + + CapturingDocumentInput capturingDocInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "hello world")), capturingDocInput); + + List> captured = capturingDocInput.getCapturedFields(); + assertTrue(captured.stream().anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("hello world"))); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatTextNullSkipped() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(b -> b.field("type", "text"))); + + CapturingDocumentInput capturingDocInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), capturingDocInput); + + List> captured = capturingDocInput.getCapturedFields(); + assertTrue(captured.stream().noneMatch(e -> e.getKey().name().equals("field"))); + } } diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java index d7c572c0cf9a9..5a61a0b9e6be0 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java @@ -53,6 +53,7 @@ import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.IndicesModule; @@ -66,8 +67,11 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.Map; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -124,6 +128,14 @@ protected final DocumentMapper createDocumentMapper(String type, String mappings return mapperService.documentMapper(); } + /** + * Create a {@link DocumentMapper} with custom index settings. + * Useful for tests that need specific settings like pluggable dataformat. + */ + protected final DocumentMapper createDocumentMapper(Settings settings, XContentBuilder mapping) throws IOException { + return createMapperService(settings, mapping).documentMapper(); + } + protected MapperService createMapperService(XContentBuilder mappings) throws IOException { return createMapperService(Version.CURRENT, mappings); } @@ -169,6 +181,42 @@ protected final MapperService createMapperService(Version version, XContentBuild return mapperService; } + /** + * Create a {@link MapperService} with custom index settings. + * Useful for tests that need specific settings like pluggable dataformat. + */ + protected final MapperService createMapperService(Settings settings, XContentBuilder mapping) throws IOException { + IndexMetadata meta = IndexMetadata.builder("index") + .settings(Settings.builder().put("index.version.created", Version.CURRENT).put(settings)) + .numberOfReplicas(0) + .numberOfShards(1) + .build(); + IndexSettings indexSettings = new IndexSettings(meta, settings); + MapperRegistry mapperRegistry = new IndicesModule( + getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList()) + ).getMapperRegistry(); + ScriptModule scriptModule = new ScriptModule( + Settings.EMPTY, + getPlugins().stream().filter(p -> p instanceof ScriptPlugin).map(p -> (ScriptPlugin) p).collect(toList()) + ); + ScriptService scriptService = new ScriptService(settings, scriptModule.engines, scriptModule.contexts); + SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, emptyMap()); + MapperService mapperService = new MapperService( + indexSettings, + createIndexAnalyzers(indexSettings), + xContentRegistry(), + similarityService, + mapperRegistry, + () -> { + throw new UnsupportedOperationException(); + }, + () -> true, + scriptService + ); + merge(mapperService, mapping); + return mapperService; + } + protected final void withLuceneIndex( MapperService mapperService, CheckedConsumer builder, @@ -301,4 +349,31 @@ protected QueryShardContext createQueryShardContext(MapperService mapperService) when(queryShardContext.documentMapper(anyString())).thenReturn(mapperService.documentMapper()); return queryShardContext; } + + /** + * A simple capturing {@link DocumentInput} that records addField calls for assertion in pluggable dataformat tests. + */ + protected static class CapturingDocumentInput implements DocumentInput { + private final List> capturedFields = new ArrayList<>(); + + @Override + public Object getFinalInput() { + return null; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + capturedFields.add(Map.entry(fieldType, value)); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public void close() {} + + public List> getCapturedFields() { + return capturedFields; + } + } } From c6653b92ebf01767db33ec2c4d6de2ac9647b551 Mon Sep 17 00:00:00 2001 From: Sagar Darji Date: Thu, 2 Apr 2026 13:33:13 +0530 Subject: [PATCH 2/3] Addressing comment for moving the pluggable dataformat check to FieldMapper class Signed-off-by: Sagar Darji --- .../index/mapper/RankFeatureFieldMapper.java | 5 + .../index/mapper/RankFeaturesFieldMapper.java | 5 + .../index/mapper/ScaledFloatFieldMapper.java | 55 +++--- .../mapper/SearchAsYouTypeFieldMapper.java | 43 +++-- .../index/mapper/TokenCountFieldMapper.java | 38 ++-- .../mapper/RankFeatureFieldMapperTests.java | 20 +++ .../mapper/RankFeaturesFieldMapperTests.java | 14 ++ .../SearchAsYouTypeFieldMapperTests.java | 34 ++++ .../join/mapper/MetaJoinFieldMapper.java | 5 + .../join/mapper/ParentIdFieldMapper.java | 18 +- .../join/mapper/ParentJoinFieldMapper.java | 8 +- .../mapper/ParentJoinFieldMapperTests.java | 170 ++++++++++++++++++ .../percolator/PercolatorFieldMapper.java | 5 + .../PercolatorFieldMapperTests.java | 13 ++ .../ICUCollationKeywordFieldMapper.java | 47 +++-- .../ICUCollationKeywordFieldMapperTests.java | 50 ++++++ .../AnnotatedTextFieldMapperTests.java | 74 ++++++++ .../AnnotatedTextFieldMapper.java | 24 ++- .../mapper/murmur3/Murmur3FieldMapper.java | 15 ++ .../murmur3/Murmur3FieldMapperTests.java | 67 +++++++ .../index/mapper/size/SizeMappingTests.java | 59 ++++++ .../org/opensearch/index/IndexSettings.java | 13 ++ .../mapper/AbstractGeometryFieldMapper.java | 5 + .../index/mapper/BinaryFieldMapper.java | 54 +++--- .../index/mapper/BooleanFieldMapper.java | 47 +++-- .../index/mapper/CompletionFieldMapper.java | 5 + .../mapper/ConstantKeywordFieldMapper.java | 9 +- .../ContextAwareGroupingFieldMapper.java | 5 + .../index/mapper/DateFieldMapper.java | 57 +++--- .../index/mapper/DerivedFieldMapper.java | 7 + .../opensearch/index/mapper/FieldMapper.java | 25 ++- .../index/mapper/FlatObjectFieldMapper.java | 57 ++++-- .../index/mapper/HllFieldMapper.java | 32 ++-- .../index/mapper/IpFieldMapper.java | 37 ++-- .../index/mapper/KeywordFieldMapper.java | 53 +++--- .../index/mapper/MetadataFieldMapper.java | 7 + .../index/mapper/NumberFieldMapper.java | 19 +- .../index/mapper/RangeFieldMapper.java | 19 +- .../mapper/SemanticVersionFieldMapper.java | 9 + .../index/mapper/StarTreeMapper.java | 11 ++ .../index/mapper/TextFieldMapper.java | 23 ++- .../index/mapper/WildcardFieldMapper.java | 33 ++-- .../index/mapper/BooleanFieldMapperTests.java | 17 ++ .../mapper/CompletionFieldMapperTests.java | 13 ++ .../ConstantKeywordFieldMapperTests.java | 30 ++++ .../ContextAwareGroupingFieldMapperTests.java | 11 ++ .../index/mapper/DerivedFieldMapperTests.java | 13 ++ .../mapper/DocumentFieldMapperTests.java | 3 + .../index/mapper/ExternalMapper.java | 5 + .../index/mapper/FakeStringFieldMapper.java | 17 ++ .../mapper/FlatObjectFieldMapperTests.java | 110 ++++++++++++ .../mapper/GeoShapeFieldMapperTests.java | 6 + .../index/mapper/HllFieldMapperTests.java | 61 +++++++ .../index/mapper/NumberFieldMapperTests.java | 13 ++ .../index/mapper/ParametrizedMapperTests.java | 5 + .../index/mapper/RangeFieldMapperTests.java | 48 +++++ .../index/mapper/RoutingFieldMapperTests.java | 15 ++ .../SemanticVersionFieldMapperTests.java | 34 ++++ .../index/mapper/StarTreeMapperTests.java | 13 ++ .../index/mapper/TextFieldMapperTests.java | 62 +++++++ .../mapper/WildcardFieldMapperTests.java | 95 ++++++++++ .../index/mapper/MockFieldMapper.java | 3 + 62 files changed, 1629 insertions(+), 241 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java index 854b1120edb49..5e348134ddf0e 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java @@ -202,6 +202,11 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addWithKey(name(), new FeatureField("_feature", name(), value)); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + // TODO check how we can support addWithKey for pluggable dataformat + } + private static Float objectToFloat(Object value) { if (value instanceof Number number) { return number.floatValue(); diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeaturesFieldMapper.java index 8c2fc705c4b52..108b1522a9ac4 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeaturesFieldMapper.java @@ -209,6 +209,11 @@ protected void parseCreateField(ParseContext context) { throw new AssertionError("parse is implemented directly"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new AssertionError("parse is implemented directly"); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index fa5f9772f735f..72d53ee7ed82f 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -451,7 +451,36 @@ protected ScaledFloatFieldMapper clone() { @Override protected void parseCreateField(ParseContext context) throws IOException { + Long scaledValue = parseScaledValue(context); + if (scaledValue == null) { + return; + } + + List fields = NumberFieldMapper.NumberType.LONG.createFields( + fieldType().name(), + scaledValue, + indexed, + hasDocValues, + skiplist, + stored + ); + context.doc().addAll(fields); + + if (hasDocValues == false && (indexed || stored)) { + createFieldNamesField(context); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + Long scaledValue = parseScaledValue(context); + if (scaledValue == null) { + return; + } + context.documentInput().addField(fieldType(), scaledValue); + } + private Long parseScaledValue(ParseContext context) throws IOException { XContentParser parser = context.parser(); Object value; Number numericValue = null; @@ -466,7 +495,7 @@ protected void parseCreateField(ParseContext context) throws IOException { numericValue = parse(parser, coerce.value()); } catch (IllegalArgumentException | JsonParseException e) { if (ignoreMalformed.value()) { - return; + return null; } else { throw e; } @@ -479,7 +508,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } if (value == null) { - return; + return null; } if (numericValue == null) { @@ -489,31 +518,13 @@ protected void parseCreateField(ParseContext context) throws IOException { double doubleValue = numericValue.doubleValue(); if (Double.isFinite(doubleValue) == false) { if (ignoreMalformed.value()) { - return; + return null; } else { // since we encode to a long, we have no way to carry NaNs and infinities throw new IllegalArgumentException("[scaled_float] only supports finite values, but got [" + doubleValue + "]"); } } - long scaledValue = Math.round(doubleValue * scalingFactor); - - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), scaledValue); - } else { - List fields = NumberFieldMapper.NumberType.LONG.createFields( - fieldType().name(), - scaledValue, - indexed, - hasDocValues, - skiplist, - stored - ); - context.doc().addAll(fields); - - if (hasDocValues == false && (indexed || stored)) { - createFieldNamesField(context); - } - } + return Math.round(doubleValue * scalingFactor); } static Double parse(Object value) { diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java index 7b973fd50b10c..ead72b615200a 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -489,6 +489,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -525,6 +530,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -650,23 +660,32 @@ public SearchAsYouTypeFieldMapper( @Override protected void parseCreateField(ParseContext context) throws IOException { - final String value = context.externalValueSet() ? context.externalValue().toString() : context.parser().textOrNull(); + final String value = extractValue(context); if (value == null) { return; } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), value); - } else { - context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); - for (ShingleFieldMapper subFieldMapper : shingleFields) { - context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); - } - context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); - if (fieldType().fieldType.omitNorms()) { - createFieldNamesField(context); - } + context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); + for (ShingleFieldMapper subFieldMapper : shingleFields) { + context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); } + context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); + if (fieldType().fieldType.omitNorms()) { + createFieldNamesField(context); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final String value = extractValue(context); + if (value == null) { + return; + } + context.documentInput().addField(fieldType(), value); + } + + private String extractValue(ParseContext context) throws IOException { + return context.externalValueSet() ? context.externalValue().toString() : context.parser().textOrNull(); } @Override diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java index f9af31c4f6fde..9dcbac8edc393 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java @@ -167,6 +167,27 @@ protected TokenCountFieldMapper( @Override protected void parseCreateField(ParseContext context) throws IOException { + final int tokenCount = parseTokenCount(context); + if (tokenCount == Integer.MIN_VALUE) { + return; + } + + context.doc() + .addAll( + NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store) + ); + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final int tokenCount = parseTokenCount(context); + if (tokenCount == Integer.MIN_VALUE) { + return; + } + context.documentInput().addField(fieldType(), tokenCount); + } + + private int parseTokenCount(ParseContext context) throws IOException { final String value; if (context.externalValueSet()) { value = context.externalValue().toString(); @@ -175,24 +196,13 @@ protected void parseCreateField(ParseContext context) throws IOException { } if (value == null && nullValue == null) { - return; + return Integer.MIN_VALUE; } - final int tokenCount; if (value == null) { - tokenCount = nullValue; - } else { - tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements); - } - - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), tokenCount); - } else { - context.doc() - .addAll( - NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, skiplist, store) - ); + return nullValue; } + return countPositions(analyzer, name(), value, enablePositionIncrements); } /** diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java index fee9471444c19..a9eada24973c3 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java @@ -38,6 +38,8 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugins.Plugin; @@ -157,4 +159,22 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage() ); } + + public void testParseCreateFieldForPluggableFormat() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + RankFeatureFieldMapper fieldMapper = (RankFeatureFieldMapper) mapper.mappers().getMapper("field"); + assertNotNull(fieldMapper); + assertEquals("rank_feature", fieldMapper.typeName()); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatRankFeatureNoOp() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", 10)), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("RankFeatureFieldMapper pluggable format is no-op", hasField); + } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java index b95572835e612..54189c0fa696a 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeaturesFieldMapperTests.java @@ -155,4 +155,18 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage() ); } + + public void testParseCreateFieldForPluggableFormat() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + RankFeaturesFieldMapper fieldMapper = (RankFeaturesFieldMapper) mapper.mappers().getMapper("field"); + assertNotNull(fieldMapper); + assertEquals("rank_features", fieldMapper.typeName()); + } + + public void testPluggableDataFormatRankFeaturesThrows() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + RankFeaturesFieldMapper rfMapper = (RankFeaturesFieldMapper) mapper.mappers().getMapper("field"); + expectThrows(AssertionError.class, () -> rfMapper.parseCreateFieldForPluggableFormat(null)); + } + } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index d5b887d70de34..6f9d9b8ba207e 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -801,4 +801,38 @@ public void testPluggableDataFormatSearchAsYouTypeNullSkipped() throws Exception boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); assertFalse("Expected no field entry for null value", found); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatSearchAsYouTypeExternalValue() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, mapping(b -> { + b.startObject("text_field"); + b.field("type", "text"); + b.startObject("fields"); + b.startObject("sayt").field("type", "search_as_you_type").endObject(); + b.endObject(); + b.endObject(); + })); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("text_field", "external_sayt")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("text_field.sayt") && e.getValue().equals("external_sayt")); + assertTrue("Expected search_as_you_type sub-field captured with external value", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatPrefixFieldMapperThrows() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject())); + PrefixFieldMapper prefixMapper = getPrefixFieldMapper(mapper, "field._index_prefix"); + expectThrows(UnsupportedOperationException.class, () -> prefixMapper.parseCreateFieldForPluggableFormat(null)); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatShingleFieldMapperThrows() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject())); + ShingleFieldMapper shingleMapper = getShingleFieldMapper(mapper, "field._2gram"); + expectThrows(UnsupportedOperationException.class, () -> shingleMapper.parseCreateFieldForPluggableFormat(null)); + } } diff --git a/modules/parent-join/src/main/java/org/opensearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/opensearch/join/mapper/MetaJoinFieldMapper.java index 2b0d3e4764b2a..8232738c5c38f 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/mapper/MetaJoinFieldMapper.java @@ -155,6 +155,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new IllegalStateException("Should never be called"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new IllegalStateException("Should never be called"); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java index 57bf6eb9ca504..d05c92e60f884 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentIdFieldMapper.java @@ -203,13 +203,19 @@ protected void parseCreateField(ParseContext context) throws IOException { } String refId = (String) context.externalValue(); BytesRef binaryValue = new BytesRef(refId); - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), binaryValue); - } else { - Field field = new Field(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + Field field = new Field(fieldType().name(), binaryValue, fieldType); + context.doc().add(field); + context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + if (context.externalValueSet() == false) { + throw new IllegalStateException("external value not set"); } + String refId = (String) context.externalValue(); + BytesRef binaryValue = new BytesRef(refId); + context.documentInput().addField(fieldType(), binaryValue); } @Override diff --git a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java index 36093d5f357e9..a1b688e1271e6 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/mapper/ParentJoinFieldMapper.java @@ -371,6 +371,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called"); + } + @Override public void parse(ParseContext context) throws IOException { context.path().add(simpleName()); @@ -433,8 +438,7 @@ public void parse(ParseContext context) throws IOException { if (isPluggableDataFormatFeatureEnabled(context)) { context.documentInput().addField(fieldType(), binaryValue); } else { - Field field = new Field(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); + context.doc().add(new Field(fieldType().name(), binaryValue, fieldType)); context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); } diff --git a/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java index ed6a8259d6e90..765236ef5a2d4 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/mapper/ParentJoinFieldMapperTests.java @@ -33,11 +33,15 @@ package org.opensearch.join.mapper; import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexService; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperException; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; @@ -47,8 +51,11 @@ import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -654,4 +661,167 @@ public void testEagerGlobalOrdinals() throws Exception { assertNotNull(service.mapperService().fieldType("join_field#child")); assertFalse(service.mapperService().fieldType("join_field#child").eagerGlobalOrdinals()); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableFormatParentDoc() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .startObject("relations") + .field("parent", "child") + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + Settings settings = Settings.builder().put("index.pluggable.dataformat.enabled", true).build(); + IndexService service = createIndex("test", settings); + DocumentMapper docMapper = service.mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + TestDocumentInput docInput = new TestDocumentInput(); + docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("join_field", "parent").endObject()), + MediaTypeRegistry.JSON + ), + docInput + ); + + // ParentJoinFieldMapper writes the join name ("parent") via documentInput + assertTrue( + "Expected join_field captured with value containing 'parent'", + docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("join_field")) + ); + // ParentIdFieldMapper writes the doc id via documentInput (parseCreateFieldForPluggableFormat) + assertTrue( + "Expected join_field#parent captured via ParentIdFieldMapper", + docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("join_field#parent")) + ); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableFormatChildDoc() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .startObject("relations") + .field("parent", "child") + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + Settings settings = Settings.builder().put("index.pluggable.dataformat.enabled", true).build(); + IndexService service = createIndex("test", settings); + DocumentMapper docMapper = service.mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + TestDocumentInput docInput = new TestDocumentInput(); + docMapper.parse( + new SourceToParse( + "test", + "2", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("join_field") + .field("name", "child") + .field("parent", "1") + .endObject() + .endObject() + ), + MediaTypeRegistry.JSON, + "1" + ), + docInput + ); + + // ParentJoinFieldMapper writes the join name ("child") via documentInput + assertTrue( + "Expected join_field captured with value containing 'child'", + docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("join_field")) + ); + // ParentIdFieldMapper writes the parent ref via documentInput (parseCreateFieldForPluggableFormat) + assertTrue( + "Expected join_field#parent captured via ParentIdFieldMapper", + docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("join_field#parent")) + ); + } + + /** + * Simple DocumentInput that captures addField calls for assertion. + */ + private static class TestDocumentInput implements DocumentInput { + private final List> capturedFields = new ArrayList<>(); + + @Override + public Object getFinalInput() { + return null; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + capturedFields.add(Map.entry(fieldType, value)); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public void close() {} + + public List> getCapturedFields() { + return capturedFields; + } + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableFormatParentJoinFieldMapperDirectThrows() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .startObject("relations") + .field("parent", "child") + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + IndexService service = createIndex("test_direct"); + DocumentMapper docMapper = service.mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + ParentJoinFieldMapper joinMapper = (ParentJoinFieldMapper) docMapper.mappers().getMapper("join_field"); + expectThrows(UnsupportedOperationException.class, () -> joinMapper.parseCreateFieldForPluggableFormat(null)); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableFormatMetaJoinFieldMapperThrows() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .startObject("relations") + .field("parent", "child") + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + IndexService service = createIndex("test_meta"); + DocumentMapper docMapper = service.mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + MetaJoinFieldMapper metaMapper = (MetaJoinFieldMapper) docMapper.mappers().getMapper("_parent_join"); + assertNotNull(metaMapper); + expectThrows(IllegalStateException.class, () -> metaMapper.parseCreateFieldForPluggableFormat(null)); + } } diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java index 05ea90eb6ce6b..be094de265701 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolatorFieldMapper.java @@ -513,6 +513,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException("should not be invoked"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + throw new UnsupportedOperationException("should not be invoked"); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java index 3a5add974c439..31107d432b67f 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorFieldMapperTests.java @@ -1172,4 +1172,17 @@ public String pluginScriptLang() { } } + public void testParseCreateFieldForPluggableFormat() throws Exception { + addQueryFieldMappings(); + DocumentMapper documentMapper = mapperService.documentMapper(); + PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); + + // parseCreateFieldForPluggableFormat should throw since PercolatorFieldMapper overrides parse() directly + UnsupportedOperationException exception = expectThrows( + UnsupportedOperationException.class, + () -> fieldMapper.parseCreateFieldForPluggableFormat(null) + ); + assertThat(exception.getMessage(), containsString("should not be invoked")); + } + } diff --git a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java index b37dfa6c40dac..82a97099a8034 100644 --- a/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -793,6 +793,33 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, @Override protected void parseCreateField(ParseContext context) throws IOException { + final BytesRef binaryValue = parseCollationKey(context); + if (binaryValue == null) { + return; + } + + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + Field field = new Field(mappedFieldType.name(), binaryValue, fieldType); + context.doc().add(field); + } + + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + createFieldNamesField(context); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final BytesRef binaryValue = parseCollationKey(context); + if (binaryValue == null) { + return; + } + context.documentInput().addField(fieldType(), binaryValue); + } + + private BytesRef parseCollationKey(ParseContext context) throws IOException { final String value; if (context.externalValueSet()) { value = context.externalValue().toString(); @@ -806,27 +833,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } if (value == null || value.length() > ignoreAbove) { - return; + return null; } RawCollationKey key = collator.getRawCollationKey(value, null); - final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size); - - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), binaryValue); - } else { - - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new Field(mappedFieldType.name(), binaryValue, fieldType); - context.doc().add(field); - } - - if (fieldType().hasDocValues()) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - createFieldNamesField(context); - } - } + return new BytesRef(key.bytes, 0, key.size); } } diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 0a2f48f4215cb..1b9f38a22d227 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -39,14 +39,19 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.plugin.analysis.icu.AnalysisICUPlugin; import org.opensearch.plugins.Plugin; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -308,4 +313,49 @@ public void testUpdateIgnoreAbove() throws IOException { assertEquals(0, fields.length); } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatCollationKeyword() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.field("field", "test_value")), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected ICU collation keyword field captured", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatCollationKeywordNullSkipped() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + private static class TestDocumentInput implements DocumentInput { + private final List> capturedFields = new ArrayList<>(); + + @Override + public Object getFinalInput() { + return null; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + capturedFields.add(Map.entry(fieldType, value)); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public void close() {} + + public List> getCapturedFields() { + return capturedFields; + } + } } diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index f8912d7760949..713b125821ddd 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -47,6 +47,8 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -58,7 +60,9 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.analysis.StandardTokenizerFactory; import org.opensearch.index.analysis.TokenFilterFactory; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MapperTestCase; @@ -68,11 +72,13 @@ import org.opensearch.plugins.Plugin; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -606,4 +612,72 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() { } } + private Settings pluggableSettings() { + return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatAnnotatedText() throws IOException { + DocumentMapper mapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.field("field", "some annotated text")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("some annotated text")); + assertTrue("Expected annotated_text field captured with value", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws IOException { + DocumentMapper mapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatWithExternalValue() throws IOException { + DocumentMapper mapper = createDocumentMapper(pluggableSettings(), mapping(b -> { + b.startObject("text_field"); + b.field("type", "text"); + b.startObject("fields"); + b.startObject("annotated").field("type", "annotated_text").endObject(); + b.endObject(); + b.endObject(); + })); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.field("text_field", "external_value")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("text_field.annotated") && e.getValue().equals("external_value")); + assertTrue("Expected annotated_text sub-field captured with external value", found); + } + + private static class TestDocumentInput implements DocumentInput { + private final List> capturedFields = new ArrayList<>(); + + @Override + public Object getFinalInput() { + return null; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + capturedFields.add(Map.entry(fieldType, value)); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public void close() {} + + public List> getCapturedFields() { + return capturedFields; + } + } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 952cff96860f2..1e696b77a1a32 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -575,13 +575,7 @@ protected AnnotatedTextFieldMapper clone() { @Override protected void parseCreateField(ParseContext context) throws IOException { - final String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); - } else { - value = context.parser().textOrNull(); - } - + final String value = getTextValue(context); if (value == null) { return; } @@ -595,6 +589,22 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final String value = getTextValue(context); + if (value == null) { + return; + } + context.documentInput().addField(fieldType(), value); + } + + private String getTextValue(ParseContext context) throws IOException { + if (context.externalValueSet()) { + return context.externalValue().toString(); + } + return context.parser().textOrNull(); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/plugins/mapper-murmur3/src/main/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapper.java index 4e87b03132055..dc67310afface 100644 --- a/plugins/mapper-murmur3/src/main/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -162,4 +162,19 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final Object value; + if (context.externalValueSet()) { + value = context.externalValue(); + } else { + value = context.parser().textOrNull(); + } + if (value != null) { + final BytesRef bytes = new BytesRef(value.toString()); + final long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1; + context.documentInput().addField(fieldType(), hash); + } + } + } diff --git a/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 04d46db50592c..3b86023548e2c 100644 --- a/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -35,17 +35,25 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.opensearch.common.hash.MurmurHash3; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperTestCase; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.plugin.mapper.MapperMurmur3Plugin; import org.opensearch.plugins.Plugin; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; public class Murmur3FieldMapperTests extends MapperTestCase { @@ -80,4 +88,63 @@ public void testDefaults() throws Exception { assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType()); } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatMurmur3() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.field("field", "test_value")), docInput); + + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected murmur3 field captured with hash value", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatMurmur3NullSkipped() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + public void testHashCalculation() throws Exception { + String testValue = "test_value"; + BytesRef bytes = new BytesRef(testValue); + long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1; + + // Verify hash is calculated (non-zero for non-empty input) + assertNotEquals("Hash should not be zero for non-empty input", 0L, hash); + + // Verify consistent hashing + BytesRef bytes2 = new BytesRef(testValue); + long hash2 = MurmurHash3.hash128(bytes2.bytes, bytes2.offset, bytes2.length, 0, new MurmurHash3.Hash128()).h1; + assertEquals("Hash should be consistent for same input", hash, hash2); + } + + private static class TestDocumentInput implements DocumentInput { + private final List> capturedFields = new ArrayList<>(); + + @Override + public Object getFinalInput() { + return null; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + capturedFields.add(Map.entry(fieldType, value)); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public void close() {} + + public List> getCapturedFields() { + return capturedFields; + } + } } diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java index 49aab68be416b..4e477803eb24f 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java @@ -35,11 +35,14 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexService; +import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.mapper.SourceToParse; @@ -48,7 +51,10 @@ import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -116,4 +122,57 @@ public void testThatDisablingWorksWhenMerging() throws Exception { assertThat(docMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(false)); } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testSizeFieldMapperPluggableFormat() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("_size") + .field("enabled", true) + .endObject() + .endObject() + .toString(); + Settings settings = Settings.builder().put("index.pluggable.dataformat.enabled", true).build(); + IndexService service = createIndex("test", settings); + DocumentMapper docMapper = service.mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + TestDocumentInput docInput = new TestDocumentInput(); + docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()), + MediaTypeRegistry.JSON + ), + docInput + ); + + // SizeFieldMapper.postParse adds the source length via documentInput when pluggable format is enabled + boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("_size")); + assertTrue("Expected _size field captured via postParse pluggable format path", found); + } + + private static class TestDocumentInput implements DocumentInput { + private final List> capturedFields = new ArrayList<>(); + + @Override + public Object getFinalInput() { + return null; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + capturedFields.add(Map.entry(fieldType, value)); + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public void close() {} + + public List> getCapturedFields() { + return capturedFields; + } + } } diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 4f9c2c9e82038..66a1c29d6a9de 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -980,6 +980,7 @@ public static IndexMergePolicy fromString(String text) { private final boolean isTranslogMetadataEnabled; private volatile boolean allowDerivedField; private final boolean derivedSourceEnabled; + private final boolean pluggableDataFormatEnabled; private volatile boolean derivedSourceEnabledForTranslog; /** @@ -1228,6 +1229,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti checkPendingFlushEnabled = scopedSettings.get(INDEX_CHECK_PENDING_FLUSH_ENABLED); defaultSearchPipeline = scopedSettings.get(DEFAULT_SEARCH_PIPELINE); derivedSourceEnabled = scopedSettings.get(INDEX_DERIVED_SOURCE_SETTING); + pluggableDataFormatEnabled = FeatureFlags.isEnabled(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + && scopedSettings.get(PLUGGABLE_DATAFORMAT_ENABLED_SETTING); derivedSourceEnabledForTranslog = scopedSettings.get(INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING); scopedSettings.addSettingsUpdateConsumer(INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING, this::setDerivedSourceEnabledForTranslog); /* There was unintentional breaking change got introduced with [OpenSearch-6424](https://github.com/opensearch-project/OpenSearch/pull/6424) (version 2.7). @@ -2367,4 +2370,14 @@ public boolean isDerivedSourceEnabledForTranslog() { public boolean isDerivedSourceEnabled() { return derivedSourceEnabled; } + + /** + * Returns whether the pluggable data format feature is enabled for this index. + * Requires both the experimental feature flag and the index-level setting. + * + * @return {@code true} if pluggable data format is enabled + */ + public boolean isPluggableDataFormatEnabled() { + return pluggableDataFormatEnabled; + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java index b19ab9590ebe6..bb6c7b47a662d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/AbstractGeometryFieldMapper.java @@ -382,6 +382,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); + } + protected abstract void addStoredFields(ParseContext context, Processed geometry); protected abstract void addDocValuesFields(String name, Processed geometry, List fields, ParseContext context); diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index cdd217e6d6eec..65c647693688b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -194,39 +194,49 @@ protected void parseCreateField(ParseContext context) throws IOException { if (stored == false && hasDocValues == false) { return; } - byte[] value = context.parseExternalValue(byte[].class); + byte[] value = parseBinaryValue(context); if (value == null) { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { - return; + return; + } + if (stored) { + context.doc().add(new StoredField(fieldType().name(), value)); + } + + if (hasDocValues) { + CustomBinaryDocValuesField field = (CustomBinaryDocValuesField) context.doc().getByKey(fieldType().name()); + if (field == null) { + field = new CustomBinaryDocValuesField(fieldType().name(), value); + context.doc().addWithKey(fieldType().name(), field); } else { - value = context.parser().binaryValue(); + field.add(value); } + } else { + // Only add an entry to the field names field if the field is stored + // but has no doc values so exists query will work on a field with + // no doc values + createFieldNamesField(context); } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + byte[] value = parseBinaryValue(context); if (value == null) { return; } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), value); - } else { - if (stored) { - context.doc().add(new StoredField(fieldType().name(), value)); - } + context.documentInput().addField(fieldType(), value); + } - if (hasDocValues) { - CustomBinaryDocValuesField field = (CustomBinaryDocValuesField) context.doc().getByKey(fieldType().name()); - if (field == null) { - field = new CustomBinaryDocValuesField(fieldType().name(), value); - context.doc().addWithKey(fieldType().name(), field); - } else { - field.add(value); - } + private byte[] parseBinaryValue(ParseContext context) throws IOException { + byte[] value = context.parseExternalValue(byte[].class); + if (value == null) { + if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + return null; } else { - // Only add an entry to the field names field if the field is stored - // but has no doc values so exists query will work on a field with - // no doc values - createFieldNamesField(context); + value = context.parser().binaryValue(); } } + return value; } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index be0c53c8fd4c4..9ce7eaf751980 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -374,6 +374,33 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } + Boolean value = parseBooleanValue(context); + if (value == null) { + return; + } + if (indexed) { + context.doc().add(new Field(fieldType().name(), value ? "T" : "F", Defaults.FIELD_TYPE)); + } + if (stored) { + context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); + } + if (hasDocValues) { + context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + } else { + createFieldNamesField(context); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + Boolean value = parseBooleanValue(context); + if (value == null) { + return; + } + context.documentInput().addField(fieldType(), value); + } + + private Boolean parseBooleanValue(ParseContext context) throws IOException { Boolean value = context.parseExternalValue(Boolean.class); if (value == null) { XContentParser.Token token = context.parser().currentToken(); @@ -385,25 +412,7 @@ protected void parseCreateField(ParseContext context) throws IOException { value = context.parser().booleanValue(); } } - - if (value == null) { - return; - } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), value); - } else { - if (indexed) { - context.doc().add(new Field(fieldType().name(), value ? "T" : "F", Defaults.FIELD_TYPE)); - } - if (stored) { - context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); - } - if (hasDocValues) { - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); - } else { - createFieldNamesField(context); - } - } + return value; } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java index 246aa2e803954..c6931caac758d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java @@ -652,6 +652,11 @@ protected void parseCreateField(ParseContext context) throws IOException { // no-op } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + // no-op + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java index 84557563d8cd3..7664e855b0254 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java @@ -252,7 +252,15 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { @Override protected void parseCreateField(ParseContext context) throws IOException { + validateConstantValue(context); + } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + validateConstantValue(context); + } + + private void validateConstantValue(ParseContext context) throws IOException { final String value; if (context.externalValueSet()) { value = context.externalValue().toString(); @@ -266,7 +274,6 @@ protected void parseCreateField(ParseContext context) throws IOException { if (!value.equals(fieldType().value)) { throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value of [" + this.value + "]"); } - } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapper.java index 458fe10acf735..bac85f932cdd8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapper.java @@ -175,6 +175,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new MapperParsingException("context_aware_grouping cannot be ingested in the document"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new MapperParsingException("context_aware_grouping cannot be ingested in the document"); + } + public ContextAwareGroupingFieldType fieldType() { return (ContextAwareGroupingFieldType) mappedFieldType; } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index bd33263594a19..dc769030adcdf 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -814,11 +814,43 @@ protected DateFieldMapper clone() { @Override protected void parseCreateField(ParseContext context) throws IOException { + Long timestamp = parseTimestamp(context); + if (timestamp == null) { + return; + } + + if (indexed) { + context.doc().add(new LongPoint(fieldType().name(), timestamp)); + } + if (hasDocValues) { + if (skiplist || isSkiplistDefaultEnabled(context.indexSettings().getIndexSortConfig(), fieldType().name())) { + context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp)); + } else { + context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); + } + } else if (store || indexed) { + createFieldNamesField(context); + } + if (store) { + context.doc().add(new StoredField(fieldType().name(), timestamp)); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + Long timestamp = parseTimestamp(context); + if (timestamp == null) { + return; + } + context.documentInput().addField(fieldType(), timestamp); + } + + private Long parseTimestamp(ParseContext context) throws IOException { String dateAsString = getFieldValue(context); long timestamp; if (dateAsString == null) { if (nullValue == null) { - return; + return null; } timestamp = nullValue; } else { @@ -827,32 +859,13 @@ protected void parseCreateField(ParseContext context) throws IOException { } catch (IllegalArgumentException | OpenSearchParseException | DateTimeException | ArithmeticException e) { if (ignoreMalformed().value()) { context.addIgnoredField(mappedFieldType.name()); - return; + return null; } else { throw e; } } } - - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), timestamp); - } else { - if (indexed) { - context.doc().add(new LongPoint(fieldType().name(), timestamp)); - } - if (hasDocValues) { - if (skiplist || isSkiplistDefaultEnabled(context.indexSettings().getIndexSortConfig(), fieldType().name())) { - context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp)); - } else { - context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); - } - } else if (store || indexed) { - createFieldNamesField(context); - } - if (store) { - context.doc().add(new StoredField(fieldType().name(), timestamp)); - } - } + return timestamp; } boolean isSkiplistDefaultEnabled(IndexSortConfig indexSortConfig, String fieldName) { diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java index d89cd6ace87b1..beb73c0de9204 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java @@ -238,6 +238,13 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("should not be invoked"); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + // Leaving this empty as the parsing should be handled via the Builder when root object is parsed. + // The context would not contain anything in this case since the DerivedFieldMapper is not indexed or stored. + throw new UnsupportedOperationException("should not be invoked"); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), this.indexAnalyzers, defaultDateFormatter, defaultIgnoreMalformed).init(this); diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index ab59867b68087..64d73616dc672 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; import org.opensearch.common.Explicit; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -60,7 +61,6 @@ import java.util.Objects; import java.util.Spliterators; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.StreamSupport; /** @@ -218,7 +218,6 @@ public T meta(Map meta) { protected MultiFields multiFields; protected CopyTo copyTo; protected DerivedFieldGenerator derivedFieldGenerator; - private final AtomicBoolean isPluggableDataFormatFeatureEnabled = new AtomicBoolean(false); protected FieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { super(simpleName); @@ -285,7 +284,11 @@ private boolean shouldIgnoreMalformed(IndexSettings is) { */ public void parse(ParseContext context) throws IOException { try { - parseCreateField(context); + if (isPluggableDataFormatFeatureEnabled(context)) { + parseCreateFieldForPluggableFormat(context); + } else { + parseCreateField(context); + } extractGroupingCriteriaParams(context); } catch (Exception e) { @@ -334,6 +337,20 @@ public void parse(ParseContext context) throws IOException { */ protected abstract void parseCreateField(ParseContext context) throws IOException; + /** + * Parse the field value and populate the pluggable data format's {@link ParseContext#documentInput()}. + *

+ * Subclasses that support pluggable data formats should override this method to extract the + * parsed value and call {@code context.documentInput().addField(fieldType(), value)}. + * The default implementation throws {@link UnsupportedOperationException}. + * + * @param context the parse context carrying the document input + * @throws IOException if an I/O error occurs while parsing + * @throws UnsupportedOperationException if the mapper does not support pluggable data formats + */ + @ExperimentalApi + protected abstract void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException; + private void extractGroupingCriteriaParams(ParseContext context) throws IOException { if (context.docMapper() != null && context.docMapper().mappers() != null) { final Mapper mapper = context.docMapper().mappers().getMapper(ContextAwareGroupingFieldMapper.CONTENT_TYPE); @@ -378,7 +395,7 @@ protected final void createFieldNamesField(ParseContext context) { } protected final boolean isPluggableDataFormatFeatureEnabled(ParseContext parseContext) { - return isPluggableDataFormatEnabled(parseContext.indexSettings().getSettings()); + return parseContext.indexSettings().isPluggableDataFormatEnabled(); } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 7b1b2615f996d..319a0677610e9 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -563,34 +563,50 @@ public FlatObjectFieldType fieldType() { @Override protected void parseCreateField(ParseContext context) throws IOException { + HashSet pathParts = parseObjectPathParts(context); + if (pathParts != null) { + createPathFields(context, pathParts); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + HashSet pathParts = parseObjectPathParts(context); + if (pathParts != null) { + createPathFieldsForPluggableFormat(context, pathParts); + } + } + + /** + * Parses the flat_object field value and returns the collected path parts, + * or {@code null} if the field should be skipped (null value or not searchable/stored/docvalues). + */ + private HashSet parseObjectPathParts(ParseContext context) throws IOException { XContentParser ctxParser = context.parser(); if (fieldType().isSearchable() == false && fieldType().isStored() == false && fieldType().hasDocValues() == false) { ctxParser.skipChildren(); - return; + return null; } - if (ctxParser.currentToken() != XContentParser.Token.VALUE_NULL) { - if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) { - throw new ParsingException( - ctxParser.getTokenLocation(), - "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value" - ); - } - parseObject(ctxParser, context); + if (ctxParser.currentToken() == XContentParser.Token.VALUE_NULL) { + return null; + } + if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + ctxParser.getTokenLocation(), + "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value" + ); } - } - private void parseObject(XContentParser parser, ParseContext context) throws IOException { - assert parser.currentToken() == XContentParser.Token.START_OBJECT; - parser.nextToken(); // Skip the outer START_OBJECT. Need to return on END_OBJECT. + assert ctxParser.currentToken() == XContentParser.Token.START_OBJECT; + ctxParser.nextToken(); LinkedList path = new LinkedList<>(Collections.singleton(fieldType().name())); HashSet pathParts = new HashSet<>(); - while (parser.currentToken() != XContentParser.Token.END_OBJECT) { - parseToken(parser, context, path, pathParts); + while (ctxParser.currentToken() != XContentParser.Token.END_OBJECT) { + parseToken(ctxParser, context, path, pathParts); } - - createPathFields(context, pathParts); + return pathParts; } private void createPathFields(ParseContext context, HashSet pathParts) { @@ -607,6 +623,13 @@ private void createPathFields(ParseContext context, HashSet pathParts) { } } + private void createPathFieldsForPluggableFormat(ParseContext context, HashSet pathParts) { + for (String part : pathParts) { + final BytesRef value = new BytesRef(name() + DOT_SYMBOL + part); + context.documentInput().addField(fieldType(), value); + } + } + private static String getDVPrefix(String rootFieldName) { return rootFieldName + DOT_SYMBOL; } diff --git a/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java index cb8841e2bdf0f..dfc210cfd19f2 100644 --- a/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/HllFieldMapper.java @@ -184,26 +184,38 @@ private HllFieldMapper(String simpleName, MappedFieldType mappedFieldType, Multi @Override protected void parseCreateField(ParseContext context) throws IOException { - // Parse binary HLL++ sketch data - byte[] value = context.parseExternalValue(byte[].class); + byte[] value = parseHllValue(context); if (value == null) { - if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { - return; - } else { - value = context.parser().binaryValue(); - } + return; } + BytesRef sketchBytes = new BytesRef(value); + validateSketchData(sketchBytes); + context.doc().add(new BinaryDocValuesField(fieldType().name(), sketchBytes)); + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + byte[] value = parseHllValue(context); if (value == null) { return; } - // Validate the sketch data BytesRef sketchBytes = new BytesRef(value); validateSketchData(sketchBytes); + context.documentInput().addField(fieldType(), value); + } - // Store as binary doc value - context.doc().add(new BinaryDocValuesField(fieldType().name(), sketchBytes)); + private byte[] parseHllValue(ParseContext context) throws IOException { + byte[] value = context.parseExternalValue(byte[].class); + if (value == null) { + if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + return null; + } else { + value = context.parser().binaryValue(); + } + } + return value; } /** diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 903391550e802..144cee9f6034b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -638,23 +638,28 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), address); - } else { - if (indexed && hasDocValues) { - context.doc().add(new InetAddressField(fieldType().name(), address)); - } else if (indexed) { - context.doc().add(new InetAddressPoint(fieldType().name(), address)); - } else if (hasDocValues) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); - } - if ((stored || indexed) && hasDocValues == false) { - createFieldNamesField(context); - } - if (stored) { - context.doc().add(new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); - } + if (indexed && hasDocValues) { + context.doc().add(new InetAddressField(fieldType().name(), address)); + } else if (indexed) { + context.doc().add(new InetAddressPoint(fieldType().name(), address)); + } else if (hasDocValues) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + } + if ((stored || indexed) && hasDocValues == false) { + createFieldNamesField(context); + } + if (stored) { + context.doc().add(new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final InetAddress address = getFieldValue(context); + if (address == null) { + return; } + context.documentInput().addField(fieldType(), address); } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index c4b0f798eb7bd..a1ed5f8547a96 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -860,6 +860,37 @@ public KeywordFieldType fieldType() { @Override protected void parseCreateField(ParseContext context) throws IOException { + String value = parseKeywordValue(context); + if (value == null) { + return; + } + + // convert to utf8 only once before feeding postings/dv/stored fields + final BytesRef binaryValue = new BytesRef(value); + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + Field field = new KeywordField(fieldType().name(), binaryValue, fieldType); + context.doc().add(field); + + if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + createFieldNamesField(context); + } + } + + if (fieldType().hasDocValues()) { + context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + String value = parseKeywordValue(context); + if (value == null) { + return; + } + context.documentInput().addField(fieldType(), value); + } + + private String parseKeywordValue(ParseContext context) throws IOException { String value; if (context.externalValueSet()) { value = context.externalValue().toString(); @@ -873,32 +904,14 @@ protected void parseCreateField(ParseContext context) throws IOException { } if (value == null || value.length() > ignoreAbove) { - return; + return null; } NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { value = normalizeValue(normalizer, name(), value); } - - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), value); - } else { - // convert to utf8 only once before feeding postings/dv/stored fields - final BytesRef binaryValue = new BytesRef(value); - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - Field field = new KeywordField(fieldType().name(), binaryValue, fieldType); - context.doc().add(field); - - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { - createFieldNamesField(context); - } - } - - if (fieldType().hasDocValues()) { - context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } - } + return value; } static String normalizeValue(NamedAnalyzer normalizer, String field, String value) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java index c98d0c8242078..5a991bb75bd02 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/MetadataFieldMapper.java @@ -205,6 +205,13 @@ protected void parseCreateField(ParseContext context) throws IOException { ); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new MapperParsingException( + "Field [" + name() + "] is a metadata field and cannot be added inside" + " a document. Use the index API request parameters." + ); + } + /** * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. */ diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index ae67161c6db7b..ecf3e536896c8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -2167,15 +2167,20 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), numericValue); - } else { - context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, skiplist, stored)); + context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, skiplist, stored)); - if (hasDocValues == false && (stored || indexed)) { - createFieldNamesField(context); - } + if (hasDocValues == false && (stored || indexed)) { + createFieldNamesField(context); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + Number numericValue = getFieldValue(context); + if (numericValue == null) { + return; } + context.documentInput().addField(fieldType(), numericValue); } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index 1a1fb2c43454c..e02ba2b1bef6c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -474,15 +474,20 @@ protected void parseCreateField(ParseContext context) throws IOException { if (range == null) { return; } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), range); - } else { - context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store)); + context.doc().addAll(fieldType().rangeType.createFields(context, name(), range, index, hasDocValues, store)); - if (hasDocValues == false && (index || store)) { - createFieldNamesField(context); - } + if (hasDocValues == false && (index || store)) { + createFieldNamesField(context); + } + } + + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + Range range = parseRange(context); + if (range == null) { + return; } + context.documentInput().addField(fieldType(), range); } private Range parseRange(ParseContext context) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java index 3e3222ff5df75..b99bd05345ce1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SemanticVersionFieldMapper.java @@ -381,6 +381,15 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + String value = context.parser().textOrNull(); + if (value == null) { + return; + } + context.documentInput().addField(fieldType(), value); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { Builder builder = new Builder(name()); diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index d1217e98749ed..543c89ec8649a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -569,6 +569,17 @@ protected void parseCreateField(ParseContext context) { ); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + throw new MapperParsingException( + String.format( + Locale.ROOT, + "Field [%s] is a star tree field and cannot be added inside a document. Use the index API request parameters.", + name() + ) + ); + } + /** * Star tree mapped field type containing dimensions, metrics, star tree specs * diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java index 2cdc5313e603b..1e7deb1dca4ed 100644 --- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java @@ -709,6 +709,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -740,6 +745,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -1040,9 +1050,7 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - if (isPluggableDataFormatFeatureEnabled(context)) { - context.documentInput().addField(fieldType(), value); - } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { Field field = new Field(fieldType().name(), value, fieldType); context.doc().add(field); if (fieldType.omitNorms()) { @@ -1057,6 +1065,15 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + final String value = getFieldValue(context); + if (value == null) { + return; + } + context.documentInput().addField(fieldType(), value); + } + @Override protected String getFieldValue(ParseContext context) throws IOException { if (context.externalValueSet()) { diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 2fd08ccfbd823..f29d8263a063a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -193,18 +193,7 @@ public int ignoreAbove() { @Override protected void parseCreateField(ParseContext context) throws IOException { - String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); - } else { - XContentParser parser = context.parser(); - if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { - value = nullValue; - } else { - value = parser.textOrNull(); - } - } - + String value = parseWildcardValue(context); if (value == null || value.length() > ignoreAbove) { return; } @@ -228,6 +217,26 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + String value = parseWildcardValue(context); + if (value == null || value.length() > ignoreAbove) { + return; + } + context.documentInput().addField(fieldType(), value); + } + + private String parseWildcardValue(ParseContext context) throws IOException { + if (context.externalValueSet()) { + return context.externalValue().toString(); + } + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + return nullValue; + } + return parser.textOrNull(); + } + /** * Tokenizer to emit tokens to support wildcard first-phase matching. *

diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 6f53da7299ffc..461cf9a8acaf4 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -357,4 +357,21 @@ public void testPluggableDataFormatBooleanNullSkipped() throws IOException { assertTrue(capturingDocInput.getCapturedFields().stream().noneMatch(e -> e.getKey().name().equals("field"))); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatBooleanNullValueConfigured() throws IOException { + Settings settings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + settings, + mapping(b -> b.startObject("field").field("type", "boolean").field("null_value", true).endObject()) + ); + CapturingDocumentInput capturingDocInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), capturingDocInput); + + assertTrue( + capturingDocInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && Boolean.TRUE.equals(e.getValue())) + ); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java index b1785f5d7b14c..99a75d58496cb 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CompletionFieldMapperTests.java @@ -46,7 +46,9 @@ import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; @@ -791,4 +793,15 @@ protected V featureValueOf(T actual) { }; } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatCompletionNoOp() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "suggestion")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("CompletionFieldMapper pluggable format is no-op, should not capture", hasField); + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java index 9fcadcfb36b69..d1c79afcb568f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java @@ -19,6 +19,7 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; @@ -164,4 +165,33 @@ private ConstantKeywordFieldMapper getMapper(FieldMapper.CopyTo copyTo) { mapper.copyTo = copyTo; return mapper; } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatConstantKeywordValidates() throws Exception { + indexService = createIndexWithSimpleMappings( + "test-pluggable", + Settings.builder().put("index.pluggable.dataformat.enabled", true).build(), + "field", + "type=constant_keyword,value=foo" + ); + ConstantKeywordFieldMapper mapper = (ConstantKeywordFieldMapper) indexService.mapperService() + .documentMapper() + .mappers() + .getMapper(FIELD_NAME); + // parseCreateFieldForPluggableFormat just validates, doesn't addField + // calling directly since it's protected and same package + // valid value should not throw + assertNotNull(mapper); + // Test via document parse — valid value should succeed + indexService.mapperService() + .documentMapper() + .parse( + new SourceToParse( + "test-pluggable", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject()), + MediaTypeRegistry.JSON + ) + ); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapperTests.java index 1a4a4210fcc9e..e9668fdcf6274 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ContextAwareGroupingFieldMapperTests.java @@ -150,6 +150,17 @@ public void testIngestAttemptThrowsException() { assertTrue(e.getMessage().contains("context_aware_grouping cannot be ingested in the document")); } + public void testPluggableDataFormatIngestThrowsException() { + ContextAwareGroupingFieldType fieldType = new ContextAwareGroupingFieldType(Collections.emptyList(), null); + ContextAwareGroupingFieldMapper mapper = new ContextAwareGroupingFieldMapper( + "context_aware_grouping", + fieldType, + new ContextAwareGroupingFieldMapper.Builder("context_aware_grouping") + ); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parseCreateFieldForPluggableFormat(null)); + assertTrue(e.getMessage().contains("context_aware_grouping cannot be ingested in the document")); + } + public void testContextAwareFieldMapperWithDerivedSource() throws IOException { ContextAwareGroupingFieldType fieldType = new ContextAwareGroupingFieldType(Collections.emptyList(), null); ContextAwareGroupingFieldMapper mapper = new ContextAwareGroupingFieldMapper( diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java index e3e2b978369ff..39e5f491569fa 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java @@ -114,4 +114,17 @@ public void testFieldInDerivedAndProperties() throws IOException { // TODO TESTCASE: testWithFieldInSource() (derived field with that field present in source) // This is more checking search behavior so may need to revisit this after query implementation + public void testPluggableDataFormatDerivedFieldThrows() throws IOException { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.startObject("derived"); + b.startObject("derived_field"); + b.field("type", "keyword"); + b.startObject("script").field("source", "emit(params._source.field)").field("lang", "painless").endObject(); + b.endObject(); + b.endObject(); + })); + DerivedFieldMapper derivedMapper = (DerivedFieldMapper) mapper.mappers().getMapper("derived_field"); + assertNotNull(derivedMapper); + expectThrows(UnsupportedOperationException.class, () -> derivedMapper.parseCreateFieldForPluggableFormat(null)); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentFieldMapperTests.java index c74b5745cd83f..1bb7d0b3bb427 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentFieldMapperTests.java @@ -104,6 +104,9 @@ static class FakeFieldMapper extends ParametrizedFieldMapper { @Override protected void parseCreateField(ParseContext context) {} + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) {} + @Override protected String contentType() { return null; diff --git a/server/src/test/java/org/opensearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/opensearch/index/mapper/ExternalMapper.java index 49b38f828f837..00c99cbd29d26 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/opensearch/index/mapper/ExternalMapper.java @@ -210,6 +210,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + throw new UnsupportedOperationException(); + } + @Override public Iterator iterator() { return Iterators.concat(super.iterator(), Arrays.asList(binMapper, boolMapper, pointMapper, shapeMapper, stringMapper).iterator()); diff --git a/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java index 9da53b36e11d6..c3fc41f3d7ced 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java @@ -122,6 +122,23 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(field); } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + String value; + if (context.externalValueSet()) { + value = context.externalValue().toString(); + } else { + value = context.parser().textOrNull(); + } + + if (value == null) { + return; + } + + Field field = new Field(fieldType().name(), value, FIELD_TYPE); + context.doc().add(field); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java index adb2c8b1ffe2a..b788a024c011e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -17,6 +17,8 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.TriFunction; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; @@ -28,6 +30,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Set; import static org.opensearch.index.mapper.FlatObjectFieldMapper.CONTENT_TYPE; @@ -417,4 +420,111 @@ protected void registerParameters(ParameterChecker checker) throws IOException { // In the future we will want to make sure parameter updates are covered. } + private Settings pluggableSettings() { + return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatSimpleObject() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "flat_object").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + String json = "{\"field\":{\"foo\":\"bar\"}}"; + mapper.parse(source(json), docInput); + + List> captured = docInput.getCapturedFields(); + boolean found = captured.stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals(new BytesRef("field.foo"))); + assertTrue("Expected flat_object path field captured", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValue() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "flat_object").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNonObjectThrows() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "flat_object").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + Exception e = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.field("field", "string_value")), docInput) + ); + assertThat(e.getCause().getMessage(), org.hamcrest.Matchers.containsString("unexpected token")); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNestedObject() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "flat_object").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + String json = "{\"field\":{\"a\":\"1\",\"b\":\"2\"}}"; + mapper.parse(source(json), docInput); + + List> captured = docInput.getCapturedFields(); + assertEquals(2, captured.size()); + Set values = Set.of(captured.get(0).getValue(), captured.get(1).getValue()); + assertTrue(values.contains(new BytesRef("field.a"))); + assertTrue(values.contains(new BytesRef("field.b"))); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatWithNullFieldsInObject() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "flat_object").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + String json = "{\"field\":{\"name\":null,\"age\":3}}"; + mapper.parse(source(json), docInput); + + List> captured = docInput.getCapturedFields(); + assertEquals(1, captured.size()); + assertEquals(new BytesRef("field.age"), captured.get(0).getValue()); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatDeepNestedObject() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "flat_object").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + String json = "{\"field\":{\"a\":{\"b\":\"val\"}}}"; + mapper.parse(source(json), docInput); + + List> captured = docInput.getCapturedFields(); + assertTrue(captured.stream().anyMatch(e -> e.getValue().equals(new BytesRef("field.a")))); + assertTrue(captured.stream().anyMatch(e -> e.getValue().equals(new BytesRef("field.b")))); + } + + public void testDefaultsDoNotUseDocumentInput() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + String json = XContentFactory.jsonBuilder() + .startObject() + .startObject("field") + .field("foo", "bar") + .endObject() + .endObject() + .toString(); + ParsedDocument doc = mapper.parse(source(json)); + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(2, fields.length); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java index 016862e3ffabc..304edb2beb8ce 100644 --- a/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/GeoShapeFieldMapperTests.java @@ -255,4 +255,10 @@ public void testGeoShapeArrayParsing() throws Exception { protected boolean supportsMeta() { return false; } + + public void testPluggableDataFormatGeoShapeThrows() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + GeoShapeFieldMapper geoMapper = (GeoShapeFieldMapper) mapper.mappers().getMapper("field"); + expectThrows(UnsupportedOperationException.class, () -> geoMapper.parseCreateFieldForPluggableFormat(null)); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java index 9dab7d1162ce3..8eb9c6a526702 100644 --- a/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/HllFieldMapperTests.java @@ -10,8 +10,10 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitMixer; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.metrics.AbstractHyperLogLog; @@ -20,6 +22,8 @@ import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Map; import static org.opensearch.search.DocValueFormat.BINARY; import static org.hamcrest.Matchers.containsString; @@ -765,4 +769,61 @@ public void testValueFetcherWithDocValues() throws IOException { // 2. Sketches can be reconstructed from doc values // 3. DocValueFormat is BINARY } + + private Settings pluggableSettings() { + return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + } + + private byte[] createValidSketchBytes() throws IOException { + HyperLogLogPlusPlus sketch = new HyperLogLogPlusPlus(HyperLogLogPlusPlus.DEFAULT_PRECISION, BigArrays.NON_RECYCLING_INSTANCE, 1); + try { + sketch.collect(0, 1L); + BytesStreamOutput out = new BytesStreamOutput(); + sketch.writeTo(0, out); + BytesRef ref = out.bytes().toBytesRef(); + return Arrays.copyOfRange(ref.bytes, ref.offset, ref.offset + ref.length); + } finally { + sketch.close(); + } + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatValidSketch() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "hll").endObject()) + ); + byte[] sketchBytes = createValidSketchBytes(); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", sketchBytes)), docInput); + + List> captured = docInput.getCapturedFields(); + boolean found = captured.stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected hll field captured with sketch bytes", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "hll").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatInvalidSketchThrows() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "hll").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + byte[] invalidBytes = new byte[] { 0, 1, 2, 3 }; + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field("field", invalidBytes)), docInput)); + assertThat(e.getMessage(), containsString("field [field]")); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java index 974ec52238f6a..15877287c170f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java @@ -789,4 +789,17 @@ public void testPluggableDataFormatDoubleValue() throws Exception { .anyMatch(e -> e.getKey().name().equals(FIELD_NAME) && e.getValue().equals(3.14)); assertTrue("Expected double value 3.14", found); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws Exception { + Settings settings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + settings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "integer").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField(FIELD_NAME)), docInput); + + assertFalse(docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(FIELD_NAME))); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ParametrizedMapperTests.java index b7c65e09bcce3..b113b773824c9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ParametrizedMapperTests.java @@ -209,6 +209,11 @@ protected void parseCreateField(ParseContext context) { } + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) { + + } + @Override protected String contentType() { return "test_mapper"; diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java index 40325d7f5b3fa..e4a2806f3fed5 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexableField; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; @@ -45,7 +46,9 @@ import java.io.IOException; import java.net.InetAddress; +import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import static org.opensearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -454,4 +457,49 @@ public void testUpdatesWithSameMappings() throws Exception { mapper.merge(mapping, MergeReason.MAPPING_UPDATE); } } + + private Settings pluggableSettings() { + return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatLongRange() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "long_range").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.startObject("field").field("gte", 5).field("lte", 10).endObject()), docInput); + + List> captured = docInput.getCapturedFields(); + boolean found = captured.stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected range field captured", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "long_range").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIpRange() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "ip_range").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "192.168.1.0/24")), docInput); + + List> captured = docInput.getCapturedFields(); + boolean found = captured.stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Expected ip_range field captured from CIDR", found); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java index 561a35efc6d18..ee57dc514a526 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldMapperTests.java @@ -33,6 +33,7 @@ package org.opensearch.index.mapper; import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -87,4 +88,18 @@ public void testIncludeInObjectNotAllowed() throws Exception { ); } } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatMetadataFieldThrows() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().endObject().toString(); + DocumentMapper docMapper = createIndex("test_pluggable").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + RoutingFieldMapper routingMapper = docMapper.metadataMapper(RoutingFieldMapper.class); + assertNotNull(routingMapper); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> routingMapper.parseCreateFieldForPluggableFormat(null) + ); + assertThat(ex.getMessage(), containsString("metadata field and cannot be added inside a document")); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java index 35bc16c512bc6..5864d5c181fa8 100644 --- a/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/SemanticVersionFieldMapperTests.java @@ -29,7 +29,9 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.geo.ShapeRelation; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -851,4 +853,36 @@ public void testFieldTypeErrorCases() { ); assertThat(fieldDataException.getMessage(), containsString("does not have doc_values enabled")); } + + private Settings pluggableSettings() { + return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatSemanticVersion() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "version").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "1.2.3")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("1.2.3")); + assertTrue("Expected version field captured with value '1.2.3'", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "version").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 96b5a133f5479..493be07c0122a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -13,6 +13,7 @@ import org.opensearch.common.Rounding; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; @@ -1496,4 +1497,16 @@ protected void writeFieldValue(XContentBuilder builder) throws IOException {} protected void registerParameters(ParameterChecker checker) throws IOException { } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatStarTreeThrows() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, getExpandedMappingWithJustAvg("status", "size")); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.field("startree", "some_value")), docInput) + ); + assertThat(ex.getCause().getMessage(), containsString("star tree field and cannot be added inside a document")); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java index bf3f7d7cd73e2..e451ea48b200a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java @@ -1199,4 +1199,66 @@ public void testPluggableDataFormatTextNullSkipped() throws IOException { List> captured = capturingDocInput.getCapturedFields(); assertTrue(captured.stream().noneMatch(e -> e.getKey().name().equals("field"))); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatTextWithExternalValue() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper(pluggableSettings, mapping(b -> { + b.startObject("text_field"); + b.field("type", "text"); + b.startObject("fields"); + b.startObject("sub").field("type", "text").endObject(); + b.endObject(); + b.endObject(); + })); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("text_field", "external_text")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("text_field.sub") && e.getValue().equals("external_text")); + assertTrue("Expected text sub-field captured with external value", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatPhraseFieldMapperThrows() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + fieldMapping(b -> b.field("type", "text").field("index_phrases", true)) + ); + TextFieldMapper textMapper = (TextFieldMapper) mapper.mappers().getMapper("field"); + Mapper phraseMapper = null; + for (Mapper m : textMapper) { + if (m.name().endsWith("._index_phrase")) { + phraseMapper = m; + break; + } + } + assertNotNull("Expected phrase sub-mapper", phraseMapper); + assertTrue(phraseMapper instanceof FieldMapper); + FieldMapper phraseFieldMapper = (FieldMapper) phraseMapper; + expectThrows(UnsupportedOperationException.class, () -> phraseFieldMapper.parseCreateFieldForPluggableFormat(null)); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatPrefixFieldMapperThrows() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + DocumentMapper mapper = createDocumentMapper( + pluggableSettings, + fieldMapping(b -> b.field("type", "text").field("index_prefixes", new java.util.HashMap<>())) + ); + TextFieldMapper textMapper = (TextFieldMapper) mapper.mappers().getMapper("field"); + Mapper prefixMapper = null; + for (Mapper m : textMapper) { + if (m.name().endsWith("._index_prefix")) { + prefixMapper = m; + break; + } + } + assertNotNull("Expected prefix sub-mapper", prefixMapper); + assertTrue(prefixMapper instanceof FieldMapper); + FieldMapper prefixFieldMapper = (FieldMapper) prefixMapper; + expectThrows(UnsupportedOperationException.class, () -> prefixFieldMapper.parseCreateFieldForPluggableFormat(null)); + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java index 3e0c88de46c5f..1f1ea622578ea 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java @@ -29,6 +29,7 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; @@ -396,4 +397,98 @@ private Document createDocument(WildcardFieldMapper mapper, String value) throws doc.add(new SortedSetDocValuesField(FIELD_NAME, binaryValue)); return doc; } + + private Settings pluggableSettings() { + return Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatDefaultWildcard() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "wildcard").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "test_value")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("test_value")); + assertTrue("Expected wildcard field captured with value 'test_value'", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueSkipped() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "wildcard").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field for null value", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatNullValueConfigured() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "wildcard").field("null_value", "default_val").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.nullField("field")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("default_val")); + assertTrue("Expected wildcard field captured with null_value 'default_val'", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIgnoreAbove() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 5).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "opensearch")), docInput); + + boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Expected no captured field when value exceeds ignore_above", hasField); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatIgnoreAboveWithinLimit() throws IOException { + DocumentMapper mapper = createDocumentMapper( + pluggableSettings(), + mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 10).endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("field", "elk")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("elk")); + assertTrue("Expected wildcard field captured with value 'elk' within ignore_above limit", found); + } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggableDataFormatWithExternalValue() throws IOException { + DocumentMapper mapper = createDocumentMapper(pluggableSettings(), mapping(b -> { + b.startObject("text_field"); + b.field("type", "text"); + b.startObject("fields"); + b.startObject("wc").field("type", "wildcard").endObject(); + b.endObject(); + b.endObject(); + })); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + mapper.parse(source(b -> b.field("text_field", "external_wildcard")), docInput); + + boolean found = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("text_field.wc") && e.getValue().equals("external_wildcard")); + assertTrue("Expected wildcard sub-field captured with external value", found); + } } diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/opensearch/index/mapper/MockFieldMapper.java index 6e504e0f8d20a..e58467cec3983 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MockFieldMapper.java @@ -91,6 +91,9 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) {} + @Override + protected void parseCreateFieldForPluggableFormat(ParseContext context) {} + public static class Builder extends ParametrizedFieldMapper.Builder { private final MappedFieldType fieldType; From 1ae69d706e542077db6f7ffd9318fc941b5d7607 Mon Sep 17 00:00:00 2001 From: Sagar Darji Date: Wed, 8 Apr 2026 12:32:03 +0530 Subject: [PATCH 3/3] Addressing comments for test cases and making parseCreateFieldForPluggableFormat as default rather than abstract Signed-off-by: Sagar Darji --- .../index/mapper/RankFeatureFieldMapper.java | 5 - .../mapper/RankFeatureFieldMapperTests.java | 11 +- .../mapper/ScaledFloatFieldMapperTests.java | 52 +++++++++ .../SearchAsYouTypeFieldMapperTests.java | 48 ++++++++ .../mapper/TokenCountFieldMapperTests.java | 36 ++++++ .../ICUCollationKeywordFieldMapperTests.java | 35 ++++++ .../AnnotatedTextFieldMapperTests.java | 36 ++++++ .../murmur3/Murmur3FieldMapperTests.java | 35 ++++++ .../opensearch/index/mapper/FieldMapper.java | 4 +- .../index/mapper/BinaryFieldMapperTests.java | 55 +++++++++ .../index/mapper/BooleanFieldMapperTests.java | 72 ++++++++++++ .../index/mapper/DateFieldMapperTests.java | 56 +++++++++ .../index/mapper/FakeStringFieldMapper.java | 26 ++--- .../index/mapper/IpFieldMapperTests.java | 52 +++++++++ .../index/mapper/KeywordFieldMapperTests.java | 106 ++++++++++++++++++ .../index/mapper/NumberFieldMapperTests.java | 74 ++++++++++++ .../index/mapper/TextFieldMapperTests.java | 56 +++++++++ .../mapper/WildcardFieldMapperTests.java | 82 ++++++++++++++ 18 files changed, 813 insertions(+), 28 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java index 5e348134ddf0e..854b1120edb49 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/RankFeatureFieldMapper.java @@ -202,11 +202,6 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addWithKey(name(), new FeatureField("_feature", name(), value)); } - @Override - protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - // TODO check how we can support addWithKey for pluggable dataformat - } - private static Float objectToFloat(Object value) { if (value instanceof Number number) { return number.floatValue(); diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java index a9eada24973c3..77d1ad63a7964 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/RankFeatureFieldMapperTests.java @@ -168,13 +168,14 @@ public void testParseCreateFieldForPluggableFormat() throws Exception { } @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) - public void testPluggableDataFormatRankFeatureNoOp() throws IOException { + public void testPluggableDataFormatRankFeatureThrows() throws IOException { Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); DocumentMapper mapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); CapturingDocumentInput docInput = new CapturingDocumentInput(); - mapper.parse(source(b -> b.field("field", 10)), docInput); - - boolean hasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); - assertFalse("RankFeatureFieldMapper pluggable format is no-op", hasField); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.field("field", 10)), docInput) + ); + assertThat(e.getCause(), instanceOf(UnsupportedOperationException.class)); } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java index d631bb142a4af..a1cc0efb6272a 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -44,6 +44,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.store.Directory; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; @@ -553,4 +554,55 @@ public void testPluggableDataFormatScaledFloatNullSkipped() throws Exception { boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); assertFalse("Expected no field entry for null value", found); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: scaled float value + assertScaledFloatLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject()), + b -> b.field("field", 3.14), + "field", + true + ); + + // Scenario 2: null value — no field produced + assertScaledFloatLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "scaled_float").field("scaling_factor", 100).endObject()), + b -> b.nullField("field"), + "field", + false + ); + } + + private void assertScaledFloatLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + boolean expectField + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + + if (!expectField) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertTrue("Pluggable path should capture field '" + fieldName + "'", pluggableHasField); + } + } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 6f9d9b8ba207e..ae7dbb12c8cd8 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -835,4 +835,52 @@ public void testPluggableDataFormatShingleFieldMapperThrows() throws Exception { ShingleFieldMapper shingleMapper = getShingleFieldMapper(mapper, "field._2gram"); expectThrows(UnsupportedOperationException.class, () -> shingleMapper.parseCreateFieldForPluggableFormat(null)); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: search_as_you_type value + { + DocumentMapper luceneMapper = createDocumentMapper( + mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) + ); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "hello world"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(b -> b.field("field", "hello world")), docInput); + + assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); + assertEquals("hello world", luceneFields[0].stringValue()); + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("hello world")); + assertTrue("Pluggable path should capture field 'field' with value 'hello world'", pluggableFound); + } + + // Scenario 2: null value — no field produced + { + DocumentMapper luceneMapper = createDocumentMapper( + mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) + ); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "search_as_you_type").endObject()) + ); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(b -> b.nullField("field")), docInput); + + assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); + } + } } diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java index 7e97421bc9228..3ad2681aa0695 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java @@ -262,4 +262,40 @@ public void testPluggableDataFormatTokenCountNullSkipped() throws Exception { boolean hasTokenCountField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc")); assertFalse("Expected no token count field for null value", hasTokenCountField); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + // Scenario 1: token count value + { + DocumentMapper luceneMapper = createIndexWithTokenCountField(false); + ParseContext.Document luceneDoc = parseDocument(luceneMapper, createDocument("three tokens string")); + IndexableField luceneField = luceneDoc.getField("test.tc"); + + DocumentMapper pluggableMapper = createIndexWithTokenCountFieldPluggableDataFormat(); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(createDocument("three tokens string"), docInput); + + assertNotNull("Lucene path should produce field 'test.tc'", luceneField); + assertEquals(3, luceneField.numericValue()); + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("test.tc") && e.getValue().equals(3)); + assertTrue("Pluggable path should capture field 'test.tc' with value 3", pluggableFound); + } + + // Scenario 2: null value — no field produced + { + DocumentMapper luceneMapper = createIndexWithTokenCountField(false); + ParseContext.Document luceneDoc = parseDocument(luceneMapper, createDocument(null)); + IndexableField luceneField = luceneDoc.getField("test.tc"); + + DocumentMapper pluggableMapper = createIndexWithTokenCountFieldPluggableDataFormat(); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(createDocument(null), docInput); + + assertNull("Lucene path should produce no field 'test.tc'", luceneField); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("test.tc")); + assertFalse("Pluggable path should produce no field 'test.tc'", pluggableHasField); + } + } } diff --git a/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 1b9f38a22d227..0e577d6181c6e 100644 --- a/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/opensearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -335,6 +335,41 @@ public void testPluggableDataFormatCollationKeywordNullSkipped() throws IOExcept assertFalse("Expected no captured field for null value", hasField); } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: collation keyword value + { + DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "1234"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + pluggableMapper.parse(source(b -> b.field("field", "1234")), docInput); + + assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); + boolean pluggableFound = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Pluggable path should capture field 'field'", pluggableFound); + } + + // Scenario 2: null value — no field produced + { + DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + pluggableMapper.parse(source(b -> b.nullField("field")), docInput); + + assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); + } + } + private static class TestDocumentInput implements DocumentInput { private final List> capturedFields = new ArrayList<>(); diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 713b125821ddd..5c84488165dfb 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/opensearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -657,6 +657,42 @@ public void testPluggableDataFormatWithExternalValue() throws IOException { assertTrue("Expected annotated_text sub-field captured with external value", found); } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + // Scenario 1: annotated text value + { + DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "some annotated text"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + pluggableMapper.parse(source(b -> b.field("field", "some annotated text")), docInput); + + assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); + assertEquals("some annotated text", luceneFields[0].stringValue()); + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals("field") && e.getValue().equals("some annotated text")); + assertTrue("Pluggable path should capture field 'field' with value 'some annotated text'", pluggableFound); + } + + // Scenario 2: null value — no field produced + { + DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings(), fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + pluggableMapper.parse(source(b -> b.nullField("field")), docInput); + + assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); + } + } + private static class TestDocumentInput implements DocumentInput { private final List> capturedFields = new ArrayList<>(); diff --git a/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 3b86023548e2c..b9f160a310fb2 100644 --- a/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/opensearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -110,6 +110,41 @@ public void testPluggableDataFormatMurmur3NullSkipped() throws IOException { assertFalse("Expected no captured field for null value", hasField); } + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: murmur3 value + { + DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.field("field", "test_value"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + pluggableMapper.parse(source(b -> b.field("field", "test_value")), docInput); + + assertTrue("Lucene path should produce field 'field'", luceneFields.length > 0); + boolean pluggableFound = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertTrue("Pluggable path should capture field 'field'", pluggableFound); + } + + // Scenario 2: null value — no field produced + { + DocumentMapper luceneMapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument luceneDoc = luceneMapper.parse(source(b -> b.nullField("field"))); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields("field"); + + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, fieldMapping(this::minimalMapping)); + TestDocumentInput docInput = new TestDocumentInput(); + pluggableMapper.parse(source(b -> b.nullField("field")), docInput); + + assertEquals("Lucene path should produce no field 'field'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); + assertFalse("Pluggable path should produce no field 'field'", pluggableHasField); + } + } + public void testHashCalculation() throws Exception { String testValue = "test_value"; BytesRef bytes = new BytesRef(testValue); diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index 64d73616dc672..850605597394b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -349,7 +349,9 @@ public void parse(ParseContext context) throws IOException { * @throws UnsupportedOperationException if the mapper does not support pluggable data formats */ @ExperimentalApi - protected abstract void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException; + protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { + throw new UnsupportedOperationException("Field mapper [" + typeName() + "] does not support pluggable data formats"); + }; private void extractGroupingCriteriaParams(ParseContext context) throws IOException { if (context.docMapper() != null && context.docMapper().mappers() != null) { diff --git a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java index cdb49d4627303..12b40a36947e3 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BinaryFieldMapperTests.java @@ -32,7 +32,9 @@ package org.opensearch.index.mapper; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -167,4 +169,57 @@ public void testPluggableDataFormatBinaryNullSkipped() throws Exception { boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); assertFalse("Expected no binary field to be captured for null value", found); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: binary value — both paths produce a field + byte[] testValue = new byte[] { 1, 2, 3 }; + String base64Value = java.util.Base64.getEncoder().encodeToString(testValue); + assertBinaryLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()), + b -> b.field("field", base64Value), + "field", + true + ); + + // Scenario 2: null value — no field produced + assertBinaryLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "binary").field("doc_values", true).endObject()), + b -> b.nullField("field"), + "field", + false + ); + } + + private void assertBinaryLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + boolean expectField + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + + if (!expectField) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertTrue("Pluggable path should capture field '" + fieldName + "'", pluggableHasField); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 461cf9a8acaf4..4386a3ccc06e6 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -47,6 +47,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Booleans; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; @@ -374,4 +375,75 @@ public void testPluggableDataFormatBooleanNullValueConfigured() throws IOExcepti .anyMatch(e -> e.getKey().name().equals("field") && Boolean.TRUE.equals(e.getValue())) ); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: true value + assertBooleanLuceneAndPluggablePathsEquivalent( + pluggableSettings, + fieldMapping(this::minimalMapping), + b -> b.field("field", true), + "field", + Boolean.TRUE + ); + + // Scenario 2: false value + assertBooleanLuceneAndPluggablePathsEquivalent( + pluggableSettings, + fieldMapping(this::minimalMapping), + b -> b.field("field", false), + "field", + Boolean.FALSE + ); + + // Scenario 3: null value — no field produced + assertBooleanLuceneAndPluggablePathsEquivalent( + pluggableSettings, + fieldMapping(this::minimalMapping), + b -> b.nullField("field"), + "field", + null + ); + + // Scenario 4: null_value configured — substitution kicks in + assertBooleanLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "boolean").field("null_value", true).endObject()), + b -> b.nullField("field"), + "field", + Boolean.TRUE + ); + } + + private void assertBooleanLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + Boolean expectedValue + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + if (expectedValue == null) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(fieldName) && expectedValue.equals(e.getValue())); + assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java index 8f714e6b30e39..54ac0a0d22f95 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java @@ -46,6 +46,7 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.util.FeatureFlags; @@ -966,4 +967,59 @@ public void testPluggableDataFormatDateNullSkipped() throws Exception { boolean hasDateField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals("field")); assertFalse("Expected no captured field for null value", hasDateField); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: date value + assertDateLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "date").endObject()), + b -> b.field("field", "2025-02-18T06:00:00.000Z"), + "field", + TEST_TIMESTAMP + ); + + // Scenario 2: null value — no field produced + assertDateLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "date").endObject()), + b -> b.nullField("field"), + "field", + null + ); + } + + private void assertDateLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + Long expectedValue + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + if (expectedValue == null) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertEquals(expectedValue.longValue(), luceneFields[0].numericValue().longValue()); + + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(fieldName) && e.getValue().equals(expectedValue)); + assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java index c3fc41f3d7ced..9e5153058d67e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/opensearch/index/mapper/FakeStringFieldMapper.java @@ -107,36 +107,28 @@ protected FakeStringFieldMapper(MappedFieldType mappedFieldType, MultiFields mul @Override protected void parseCreateField(ParseContext context) throws IOException { - String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); - } else { - value = context.parser().textOrNull(); - } - + String value = parseValue(context); if (value == null) { return; } - Field field = new Field(fieldType().name(), value, FIELD_TYPE); context.doc().add(field); } @Override protected void parseCreateFieldForPluggableFormat(ParseContext context) throws IOException { - String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); - } else { - value = context.parser().textOrNull(); - } - + String value = parseValue(context); if (value == null) { return; } + context.documentInput().addField(fieldType(), value); + } - Field field = new Field(fieldType().name(), value, FIELD_TYPE); - context.doc().add(field); + private String parseValue(ParseContext context) throws IOException { + if (context.externalValueSet()) { + return context.externalValue().toString(); + } + return context.parser().textOrNull(); } @Override diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java index 1e1bc541ef3cc..ba101d7d2b25c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldMapperTests.java @@ -46,6 +46,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -347,4 +348,55 @@ public void testPluggableDataFormatIpNullSkipped() throws Exception { boolean found = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(FIELD_NAME)); assertFalse("Expected no ip field to be captured for null value", found); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: ip value + assertIpLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "ip").endObject()), + b -> b.field(FIELD_NAME, "192.168.1.1"), + FIELD_NAME, + true + ); + + // Scenario 2: null value — no field produced + assertIpLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "ip").endObject()), + b -> b.nullField(FIELD_NAME), + FIELD_NAME, + false + ); + } + + private void assertIpLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + boolean expectField + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + + if (!expectField) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertTrue("Pluggable path should capture field '" + fieldName + "'", pluggableHasField); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java index 30f4df625b371..993a8af11075f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldMapperTests.java @@ -51,6 +51,7 @@ import org.apache.lucene.tests.analysis.MockLowerCaseFilter; import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; @@ -707,4 +708,109 @@ public void testDefaultsDoNotUseDocumentInput() throws Exception { assertEquals(new BytesRef("1234"), fields[0].binaryValue()); assertEquals(new BytesRef("1234"), fields[1].binaryValue()); } + + /** + * Cross-path equivalence test: verifies that the pluggable DocumentInput path + * captures the same field values as the Lucene Document path for all common + * keyword scenarios (default, null_value, ignore_above, normalizer, multi-field). + */ + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: default keyword + assertLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").endObject()), + b -> b.field("field", "1234"), + "field", + "1234" + ); + + // Scenario 2: null value — no field produced + assertLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").endObject()), + b -> b.nullField("field"), + "field", + null + ); + + // Scenario 3: null_value configured — substitution kicks in + assertLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("null_value", "uri").endObject()), + b -> b.nullField("field"), + "field", + "uri" + ); + + // Scenario 4: ignore_above — value exceeds limit, no field produced + assertLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 5).endObject()), + b -> b.field("field", "opensearch"), + "field", + null + ); + + // Scenario 5: ignore_above — value within limit + assertLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("ignore_above", 5).endObject()), + b -> b.field("field", "elk"), + "field", + "elk" + ); + + // Scenario 6: normalizer + assertLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject("field").field("type", "keyword").field("normalizer", "lowercase").endObject()), + b -> b.field("field", "AbC"), + "field", + "abc" + ); + } + + /** + * Parses the same source through both the Lucene path and the pluggable DocumentInput path, + * then asserts they agree on the produced field value (or absence thereof). + * + * @param expectedValue the expected value, or null if no field should be produced + */ + private void assertLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + String expectedValue + ) throws IOException { + // Lucene path (default, no pluggable setting) + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + if (expectedValue == null) { + // Both paths should produce no field + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + // Lucene path should have produced the field with the expected value + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertEquals(new BytesRef(expectedValue), luceneFields[0].binaryValue()); + + // Pluggable path should capture the same value + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(fieldName) && e.getValue().equals(expectedValue)); + assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java index 15877287c170f..b3fc834501f05 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldMapperTests.java @@ -49,6 +49,7 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.store.Directory; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Numbers; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -802,4 +803,77 @@ public void testPluggableDataFormatNullValueSkipped() throws Exception { assertFalse(docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(FIELD_NAME))); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws Exception { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: integer value + assertNumericLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "integer").endObject()), + b -> b.field(FIELD_NAME, 42), + FIELD_NAME, + 42 + ); + + // Scenario 2: long value + assertNumericLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "long").endObject()), + b -> b.field(FIELD_NAME, 123456789L), + FIELD_NAME, + 123456789L + ); + + // Scenario 3: double value + assertNumericLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "double").endObject()), + b -> b.field(FIELD_NAME, 3.14), + FIELD_NAME, + 3.14 + ); + + // Scenario 4: null value — no field produced + assertNumericLuceneAndPluggablePathsEquivalent( + pluggableSettings, + mapping(b -> b.startObject(FIELD_NAME).field("type", "integer").endObject()), + b -> b.nullField(FIELD_NAME), + FIELD_NAME, + null + ); + } + + private void assertNumericLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + Number expectedValue + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + if (expectedValue == null) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertEquals(expectedValue.doubleValue(), luceneFields[0].numericValue().doubleValue(), 0.001d); + + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(fieldName) && e.getValue().equals(expectedValue)); + assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java index e451ea48b200a..b0a76b0599a07 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldMapperTests.java @@ -72,6 +72,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -1261,4 +1262,59 @@ public void testPluggableDataFormatPrefixFieldMapperThrows() throws IOException FieldMapper prefixFieldMapper = (FieldMapper) prefixMapper; expectThrows(UnsupportedOperationException.class, () -> prefixFieldMapper.parseCreateFieldForPluggableFormat(null)); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + Settings pluggableSettings = Settings.builder().put(getIndexSettings()).put("index.pluggable.dataformat.enabled", true).build(); + + // Scenario 1: default text value + assertTextLuceneAndPluggablePathsEquivalent( + pluggableSettings, + fieldMapping(b -> b.field("type", "text")), + b -> b.field("field", "hello world"), + "field", + "hello world" + ); + + // Scenario 2: null value — no field produced + assertTextLuceneAndPluggablePathsEquivalent( + pluggableSettings, + fieldMapping(b -> b.field("type", "text")), + b -> b.nullField("field"), + "field", + null + ); + } + + private void assertTextLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + String expectedValue + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + if (expectedValue == null) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + assertEquals(expectedValue, luceneFields[0].stringValue()); + + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(fieldName) && e.getValue().equals(expectedValue)); + assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); + } + } } diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java index 1f1ea622578ea..29da0f00ac2a9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldMapperTests.java @@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; @@ -491,4 +492,85 @@ public void testPluggableDataFormatWithExternalValue() throws IOException { .anyMatch(e -> e.getKey().name().equals("text_field.wc") && e.getValue().equals("external_wildcard")); assertTrue("Expected wildcard sub-field captured with external value", found); } + + @LockFeatureFlag(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG) + public void testPluggablePathEquivalenceWithLucenePath() throws IOException { + Settings pluggable = pluggableSettings(); + + // Scenario 1: default wildcard value + assertWildcardLuceneAndPluggablePathsEquivalent( + pluggable, + mapping(b -> b.startObject("field").field("type", "wildcard").endObject()), + b -> b.field("field", "test_value"), + "field", + "test_value" + ); + + // Scenario 2: null value — no field produced + assertWildcardLuceneAndPluggablePathsEquivalent( + pluggable, + mapping(b -> b.startObject("field").field("type", "wildcard").endObject()), + b -> b.nullField("field"), + "field", + null + ); + + // Scenario 3: null_value configured — substitution kicks in + assertWildcardLuceneAndPluggablePathsEquivalent( + pluggable, + mapping(b -> b.startObject("field").field("type", "wildcard").field("null_value", "default_val").endObject()), + b -> b.nullField("field"), + "field", + "default_val" + ); + + // Scenario 4: ignore_above — value exceeds limit, no field produced + assertWildcardLuceneAndPluggablePathsEquivalent( + pluggable, + mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 5).endObject()), + b -> b.field("field", "opensearch"), + "field", + null + ); + + // Scenario 5: ignore_above — value within limit + assertWildcardLuceneAndPluggablePathsEquivalent( + pluggable, + mapping(b -> b.startObject("field").field("type", "wildcard").field("ignore_above", 10).endObject()), + b -> b.field("field", "elk"), + "field", + "elk" + ); + } + + private void assertWildcardLuceneAndPluggablePathsEquivalent( + Settings pluggableSettings, + XContentBuilder mappingBuilder, + CheckedConsumer sourceBuilder, + String fieldName, + String expectedValue + ) throws IOException { + // Lucene path + DocumentMapper luceneMapper = createDocumentMapper(mappingBuilder); + ParsedDocument luceneDoc = luceneMapper.parse(source(sourceBuilder)); + IndexableField[] luceneFields = luceneDoc.rootDoc().getFields(fieldName); + + // Pluggable path + DocumentMapper pluggableMapper = createDocumentMapper(pluggableSettings, mappingBuilder); + CapturingDocumentInput docInput = new CapturingDocumentInput(); + pluggableMapper.parse(source(sourceBuilder), docInput); + + if (expectedValue == null) { + assertEquals("Lucene path should produce no field for '" + fieldName + "'", 0, luceneFields.length); + boolean pluggableHasField = docInput.getCapturedFields().stream().anyMatch(e -> e.getKey().name().equals(fieldName)); + assertFalse("Pluggable path should produce no field for '" + fieldName + "'", pluggableHasField); + } else { + assertTrue("Lucene path should produce field '" + fieldName + "'", luceneFields.length > 0); + + boolean pluggableFound = docInput.getCapturedFields() + .stream() + .anyMatch(e -> e.getKey().name().equals(fieldName) && e.getValue().equals(expectedValue)); + assertTrue("Pluggable path should capture field '" + fieldName + "' with value '" + expectedValue + "'", pluggableFound); + } + } }