From 64001341cb0c0574dd9c6cf45eafbb86b8d10c3d Mon Sep 17 00:00:00 2001 From: Zhiyong Yang Date: Sun, 22 Mar 2020 11:42:46 +0800 Subject: [PATCH 1/3] add httpRequestTimeout to odata based connector activities --- .../2018-06-01/entityTypes/Pipeline.json | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 28298bcea8a4..b4e992ccebf6 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1535,6 +1535,10 @@ "type": "object", "description": "OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + }, "additionalColumns": { "type": "array", "description": "Specifies the additional columns to be added to source data. Type: array of objects (or Expression with resultType array of objects).", @@ -1617,6 +1621,10 @@ "query": { "type": "object", "description": "SAP Cloud for Customer OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -1632,6 +1640,10 @@ "query": { "type": "object", "description": "SAP ECC OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -2990,6 +3002,10 @@ "query": { "type": "object", "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -3209,6 +3225,10 @@ "name": "SapCloudForCustomerSinkWriteBehavior", "modelAsString": true } + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, From 51f202dffa40ad952df773322b441127b30acd51 Mon Sep 17 00:00:00 2001 From: Zhiyong Yang Date: Sun, 22 Mar 2020 11:47:19 +0800 Subject: [PATCH 2/3] add httpRequestTimeout to odata based connector activities --- .../stable/2018-06-01/entityTypes/Pipeline.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index b4e992ccebf6..9c19f931449f 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1537,7 +1537,7 @@ }, "httpRequestTimeout": { "type": "object", - "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." }, "additionalColumns": { "type": "array", @@ -1624,7 +1624,7 @@ }, "httpRequestTimeout": { "type": "object", - "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -1643,7 +1643,7 @@ }, "httpRequestTimeout": { "type": "object", - "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -3005,7 +3005,7 @@ }, "httpRequestTimeout": { "type": "object", - "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -3228,7 +3228,7 @@ }, "httpRequestTimeout": { "type": "object", - "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:01:40. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, From db65013ae5a5e81490c5b27669e0872b2527011b Mon Sep 17 00:00:00 2001 From: Zhiyong Yang Date: Thu, 4 Jun 2020 15:10:23 +0800 Subject: [PATCH 3/3] add sql parallel read properties --- .../2018-06-01/entityTypes/Pipeline.json | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 3f87eba0a425..bc89fb5b7d7b 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -2111,6 +2111,14 @@ "isolationLevel": { "description": "Specifies the transaction locking behavior for the SQL source. Allowed values: ReadCommitted/ReadUncommitted/RepeatableRead/Serializable/Snapshot. The default value is ReadCommitted. Type: string (or Expression with resultType string).", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2141,6 +2149,14 @@ "produceAdditionalTypes": { "description": "Which additional types to produce.", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2171,6 +2187,14 @@ "produceAdditionalTypes": { "description": "Which additional types to produce.", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2201,6 +2225,14 @@ "produceAdditionalTypes": { "description": "Which additional types to produce.", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2224,9 +2256,48 @@ "storedProcedureParameters": { "type": "object", "description": "Value and type setting for stored procedure parameters. Example: \"{Parameter1: {value: \"1\", type: \"int\"}}\". Type: object (or Expression with resultType object), itemType: StoredProcedureParameter." + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, + "SqlPartitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "type": "object", + "properties": { + "partitionColumnName": { + "type": "object", + "description": "The name of the column in integer or datetime type that will be used for proceeding partitioning. If not specified, the primary key of the table is auto-detected and used as the partition column. Type: string (or Expression with resultType string)." + }, + "partitionUpperBound": { + "type": "object", + "description": "The maximum value of the partition column for partition range splitting. This value is used to decide the partition stride, not for filtering the rows in table. All rows in the table or query result will be partitioned and copied. Type: string (or Expression with resultType string)." + }, + "partitionLowerBound": { + "type": "object", + "description": "The minimum value of the partition column for partition range splitting. This value is used to decide the partition stride, not for filtering the rows in table. All rows in the table or query result will be partitioned and copied. Type: string (or Expression with resultType string)." + } + } + }, + "SqlPartitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "type": "string", + "enum": [ + "None", + "PhysicalPartitionsOfTable", + "DynamicRange" + ], + "x-ms-enum": { + "name": "SqlPartitionOption", + "modelAsString": true + } + }, "FileSystemSource": { "description": "A copy activity file system source.", "type": "object",