Add mapper_type and mapper_settings to IngestionSource schema

CHANGELOG.md

@@ -42,6 +42,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `agentic` query type, `agentic_query_translator` request processor, and `agentic_context` response processor ([#1073](https://github.com/opensearch-project/opensearch-api-specification/pull/1073))
 - Added `allow_no_indices`, `ignore_unavailable`, and `ignore_throttled` query parameters to `create_pit` ([#1141](https://github.com/opensearch-project/opensearch-api-specification/pull/1141))
 - Added `query` to TermsLookup to support terms lookup by query
+- Added `mapper_type` and `mapper_settings` to `IngestionSource` index settings ([#1155](https://github.com/opensearch-project/opensearch-api-specification/pull/1155))
 
 ### Deprecated
 - Marked the plural `_aliases` URL forms of `put_alias` and `delete_alias` as deprecated; the singular `_alias` form is the canonical path ([#1131](https://github.com/opensearch-project/opensearch-api-specification/pull/1131))

spec/schemas/indices._common.yaml

@@ -175,6 +175,13 @@ components:
         - kafka
         - kinesis
         - none
+    IngestionSourceMapperType:
+      type: string
+      description: The type of message mapper used to extract document fields from ingestion messages.
+      enum:
+        - default
+        - field_mapping
+        - raw_payload
     IngestionSource:
       type: object
       description: The pull-based ingestion settings controlling how data is read from streaming sources.
@@ -213,6 +220,13 @@ components:
         all_active:
           description: Indicates if all-active ingestion is enabled for this index.
           $ref: '_common.yaml#/components/schemas/StringifiedBoolean'
+        mapper_type:
+          description: The type of message mapper used to extract document fields from ingestion messages.
+          $ref: '#/components/schemas/IngestionSourceMapperType'
+        mapper_settings:
+          type: object
+          description: Configuration settings for the selected mapper type.
+          additionalProperties: true
     IngestionSourcePointer:
       type: object
       properties:

tests/plugins/ingestion-fs/pull_based_ingestion/mapper_settings.yaml

@@ -0,0 +1,64 @@
+$schema: ../../../../json_schemas/test_story.schema.yaml
+
+description: Test pull-based ingestion with mapper_type and mapper_settings.
+version: '>=3.6'
+prologues: []
+epilogues:
+  - path: /test-mapper-index
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Create an index with mapper_type and mapper_settings.
+    path: /{index}
+    method: PUT
+    parameters:
+      index: test-mapper-index
+    request:
+      payload:
+        settings:
+          ingestion_source:
+            type: file
+            mapper_type: field_mapping
+            mapper_settings:
+              id_field: my_id
+              version_field: my_version
+            pointer.init.reset: reset_by_offset
+            pointer.init.reset.value: 0
+            error_strategy: DROP
+            poll.max_batch_size: 10
+            poll.timeout: 100000
+            num_processor_threads: 1
+            internal_queue_size: 100
+            param:
+              stream: test-stream
+              base_directory: ingestion_directory
+          index:
+            number_of_shards: 1
+            number_of_replicas: 0
+            replication.type: SEGMENT
+        mappings:
+          properties:
+            name:
+              type: text
+    response:
+      status: 200
+      payload:
+        acknowledged: true
+        shards_acknowledged: true
+  - synopsis: Verify mapper_type and mapper_settings are returned in index settings.
+    path: /{index}
+    method: GET
+    parameters:
+      index: test-mapper-index
+    response:
+      status: 200
+      payload:
+        test-mapper-index:
+          settings:
+            index:
+              ingestion_source:
+                type: file
+                mapper_type: field_mapping
+                mapper_settings:
+                  id_field: my_id
+                  version_field: my_version