meltano.yml

version: 1
send_anonymous_usage_stats: true
project_id: tap-mongodb
default_environment: test
environments:
  - name: test
plugins:
  extractors:
    - name: tap-mongodb
      namespace: tap_mongodb
      pip_url: -e .
      capabilities:
        - state
        - catalog
        - discover
        - about
        - stream-maps
      config:
        database: test_database
        mongodb_connection_string: mongodb://admin:password@localhost:27017/
      settings:
        - name: database
          kind: string
          description: Database name from which records will be extracted.
        - name: mongodb_connection_string
          kind: password
          description: |
            MongoDB connection string. See
            https://www.mongodb.com/docs/manual/reference/connection-string/#connection-string-uri-format for specification.
            The password included in this string should be url-encoded. The tap will not url-encode it.
        - name: documentdb_credential_json_string
          kind: password
          description: |
            String (serialized JSON object) with keys 'username', 'password', 'engine', 'host', 'port',
            'dbClusterIdentifier' or 'dbName', 'ssl'. See example at
            https://docs.aws.amazon.com/secretsmanager/latest/userguide/reference_secret_json_structure.html#reference_secret_json_structure_docdb.
            The password from this JSON object will be url-encoded by the tap before opening the database connection.
        - name: documentdb_credential_json_extra_options
          kind: string
          description: |
            String (serialized JSON object) containing string-string key-value pairs which will be added to the
            connection string options when using documentdb_credential_json_string. For example, when set to
            the string `{\"tls\":\"true\",\"tlsCAFile\":\"my-ca-bundle.pem\"}`, the options
            `tls=true&tlsCAFile=my-ca-bundle.pem` will be passed to the MongoClient.
        - name: datetime_conversion
          kind: string
          description: |
            Parameter passed to MongoClient 'datetime_conversion' parameter. See documentation at
            https://pymongo.readthedocs.io/en/stable/examples/datetimes.html#handling-out-of-range-datetimes
            for details. The default value is 'datetime', which will throw a bson.errors.InvalidBson error
            if a document contains a date outside the range of datetime.MINYEAR (year 1) to datetime.MAXYEAR
            (9999). The allowed values correspond to the enumeration members here:
            https://github.com/mongodb/mongo-python-driver/blob/e23eb7691e6e2905a77fc39a114d000ddf057e47/bson/codec_options.py#L192-L224
            (they will be uppercased by the tap).
          value: datetime
        - name: prefix
          kind: string
          description: |
            An optional prefix which will be added to each stream name.
          value: ''
        - name: start_date
          kind: date_iso8601
          description: |
            Start date - used for incremental replication only. Log based replication does not support this setting - do
            not provide it unless you are using incremental replication. Defaults to epoch zero time `1970-01-01` if
            tap uses incremental replication method.
        - name: add_record_metadata
          kind: boolean
          description: |
            When True, _sdc metadata fields will be added to records produced by this tap.
        - name: allow_modify_change_streams
          kind: boolean
          description: |
            In DocumentDB (unlike MongoDB), change streams must be enabled specifically (see
            https://docs.aws.amazon.com/documentdb/latest/developerguide/change_streams.html#change_streams-enabling
            ). If attempting to open a change stream against a collection on which change streams have not been
            enabled, an OperationFailure error will be raised. If this property is set to True, when this error
            is seen, the tap will execute an admin command to enable change streams and then retry the read
            operation. Note: this may incur new costs in AWS DocumentDB.
        - name: operation_types
          kind: array
          description: |
            List of MongoDB change stream operation types to include in tap output. The default behavior is to
            limit to document-level operation types. See full list of operation types at
            https://www.mongodb.com/docs/manual/reference/change-events/#operation-types. Note that the list
            of allowed_values for this property includes some values not available to all MongoDB versions.
          value:
            - create
            - delete
            - insert
            - replace
            - update
        - name: stream_maps
          kind: object
          description: |
            Stream maps. See https://sdk.meltano.com/en/latest/stream_maps.html for documentation.
        - name: stream_map_config
          kind: object
          description: |
            Stream map config. See https://sdk.meltano.com/en/latest/stream_maps.html for documentation.
      settings_group_validation:
        # yamllint disable rule:hyphens
        -   -
              mongodb_connection_string
            - database
        -   - documentdb_credential_json_string
            - database
        -   - documentdb_credential_json_string
            - documentdb_credential_json_extra_options
            - database
      # yamllint enable rule:hyphens
      select:
        - '*.*'
      metadata:
        '*':
          replication-key: replication_key
          replication-method: INCREMENTAL
  loaders:
    - name: target-jsonl
      variant: andyh1203
      pip_url: target-jsonl