Skip to content

Add Amazon S3 Compatible and Oracle Cloud Storage connectors #13368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
qianwens merged 1 commit into from
Mar 12, 2021
+227 −1
Merged

Add Amazon S3 Compatible and Oracle Cloud Storage connectors #13368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions ...factory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,44 @@
}
]
},
"AmazonS3CompatibleLocation": {
"description": "The location of Amazon S3 Compatible dataset.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/DatasetLocation"
}
],
"properties": {
"bucketName": {
"type": "object",
"description": "Specify the bucketName of Amazon S3 Compatible. Type: string (or Expression with resultType string)"
},
"version": {
"type": "object",
"description": "Specify the version of Amazon S3 Compatible. Type: string (or Expression with resultType string)."
}
}
},
"OracleCloudStorageLocation": {
"description": "The location of Oracle Cloud Storage dataset.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/DatasetLocation"
}
],
"properties": {
"bucketName": {
"type": "object",
"description": "Specify the bucketName of Oracle Cloud Storage. Type: string (or Expression with resultType string)"
},
"version": {
"type": "object",
"description": "Specify the version of Oracle Cloud Storage. Type: string (or Expression with resultType string)."
}
}
},
"GoogleCloudStorageLocation": {
"description": "The location of Google Cloud Storage dataset.",
"type": "object",
Expand Down
86 changes: 86 additions & 0 deletions ...y/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1042,6 +1042,92 @@
}
}
},
"AmazonS3CompatibleLinkedService": {
"x-ms-discriminator-value": "AmazonS3Compatible",
"description": "Linked service for Amazon S3 Compatible.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/LinkedService"
}
],
"properties": {
"typeProperties": {
"description": "Amazon S3 Compatible linked service properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/AmazonS3CompatibleLinkedServiceTypeProperties"
}
},
"required": [
"typeProperties"
]
},
"AmazonS3CompatibleLinkedServiceTypeProperties": {
"description": "Amazon S3 Compatible linked service properties.",
"properties": {
"accessKeyId": {
"type": "object",
"description": "The access key identifier of the Amazon S3 Compatible Identity and Access Management (IAM) user. Type: string (or Expression with resultType string)."
},
"secretAccessKey": {
"description": "The secret access key of the Amazon S3 Compatible Identity and Access Management (IAM) user.",
"$ref": "../datafactory.json#/definitions/SecretBase"
},
"serviceUrl": {
"type": "object",
"description": "This value specifies the endpoint to access with the Amazon S3 Compatible Connector. This is an optional property; change it only if you want to try a different service endpoint or want to switch between https and http. Type: string (or Expression with resultType string)."
},
"forcePathStyle": {
"type": "object",
"description": "If true, use S3 path-style access instead of virtual hosted-style access. Default value is false. Type: boolean (or Expression with resultType boolean)."
},
"encryptedCredential": {
"type": "object",
"description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)."
}
}
},
"OracleCloudStorageLinkedService": {
"x-ms-discriminator-value": "OracleCloudStorage",
"description": "Linked service for Oracle Cloud Storage.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/LinkedService"
}
],
"properties": {
"typeProperties": {
"description": "Oracle Cloud Storage linked service properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/OracleCloudStorageLinkedServiceTypeProperties"
}
},
"required": [
"typeProperties"
]
},
"OracleCloudStorageLinkedServiceTypeProperties": {
"description": "Oracle Cloud Storage linked service properties.",
"properties": {
"accessKeyId": {
"type": "object",
"description": "The access key identifier of the Oracle Cloud Storage Identity and Access Management (IAM) user. Type: string (or Expression with resultType string)."
},
"secretAccessKey": {
"description": "The secret access key of the Oracle Cloud Storage Identity and Access Management (IAM) user.",
"$ref": "../datafactory.json#/definitions/SecretBase"
},
"serviceUrl": {
"type": "object",
"description": "This value specifies the endpoint to access with the Oracle Cloud Storage Connector. This is an optional property; change it only if you want to try a different service endpoint or want to switch between https and http. Type: string (or Expression with resultType string)."
},
"encryptedCredential": {
"type": "object",
"description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)."
}
}
},
"GoogleCloudStorageLinkedService": {
"x-ms-discriminator-value": "GoogleCloudStorage",
"description": "Linked service for Google Cloud Storage.",
Expand Down
104 changes: 103 additions & 1 deletion ...actory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@
}
},
"AmazonS3ReadSettings": {
"description": "Azure data lake store read settings.",
"description": "Amazon S3 read settings.",
"type": "object",
"allOf": [
{
Expand Down Expand Up @@ -592,6 +592,108 @@
}
}
},
"AmazonS3CompatibleReadSettings": {
"description": "Amazon S3 Compatible read settings.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/StoreReadSettings"
}
],
"properties": {
"recursive": {
"type": "object",
"description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)."
},
"wildcardFolderPath": {
"type": "object",
"description": "Amazon S3 Compatible wildcardFolderPath. Type: string (or Expression with resultType string)."
},
"wildcardFileName": {
"type": "object",
"description": "Amazon S3 Compatible wildcardFileName. Type: string (or Expression with resultType string)."
},
"prefix": {
"type": "object",
"description": "The prefix filter for the S3 Compatible object name. Type: string (or Expression with resultType string)."
},
"fileListPath": {
"type": "object",
"description": "Point to a text file that lists each file (relative path to the path configured in the dataset) that you want to copy. Type: string (or Expression with resultType string)."
},
"enablePartitionDiscovery": {
"type": "boolean",
"description": "Indicates whether to enable partition discovery."
},
"partitionRootPath": {
"type": "object",
"description": "Specify the root path where partition discovery starts from. Type: string (or Expression with resultType string)."
},
"deleteFilesAfterCompletion": {
"type": "object",
"description": "Indicates whether the source files need to be deleted after copy completion. Default is false. Type: boolean (or Expression with resultType boolean)."
},
"modifiedDatetimeStart": {
"type": "object",
"description": "The start of file's modified datetime. Type: string (or Expression with resultType string)."
},
"modifiedDatetimeEnd": {
"type": "object",
"description": "The end of file's modified datetime. Type: string (or Expression with resultType string)."
}
}
},
"OracleCloudStorageReadSettings": {
"description": "Oracle Cloud Storage read settings.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/StoreReadSettings"
}
],
"properties": {
"recursive": {
"type": "object",
"description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)."
},
"wildcardFolderPath": {
"type": "object",
"description": "Oracle Cloud Storage wildcardFolderPath. Type: string (or Expression with resultType string)."
},
"wildcardFileName": {
"type": "object",
"description": "Oracle Cloud Storage wildcardFileName. Type: string (or Expression with resultType string)."
},
"prefix": {
"type": "object",
"description": "The prefix filter for the Oracle Cloud Storage object name. Type: string (or Expression with resultType string)."
},
"fileListPath": {
"type": "object",
"description": "Point to a text file that lists each file (relative path to the path configured in the dataset) that you want to copy. Type: string (or Expression with resultType string)."
},
"enablePartitionDiscovery": {
"type": "boolean",
"description": "Indicates whether to enable partition discovery."
},
"partitionRootPath": {
"type": "object",
"description": "Specify the root path where partition discovery starts from. Type: string (or Expression with resultType string)."
},
"deleteFilesAfterCompletion": {
"type": "object",
"description": "Indicates whether the source files need to be deleted after copy completion. Default is false. Type: boolean (or Expression with resultType boolean)."
},
"modifiedDatetimeStart": {
"type": "object",
"description": "The start of file's modified datetime. Type: string (or Expression with resultType string)."
},
"modifiedDatetimeEnd": {
"type": "object",
"description": "The end of file's modified datetime. Type: string (or Expression with resultType string)."
}
}
},
"GoogleCloudStorageReadSettings": {
"description": "Google Cloud Storage read settings.",
"type": "object",
Expand Down