Skip to content

[Datafactory] Add AzureMLExecutePipeline activity and AzureMLService linked service #7395

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anuchandy merged 1 commit into from
Oct 9, 2019
Merged

[Datafactory] Add AzureMLExecutePipeline activity and AzureMLService linked service #7395

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 65 additions & 7 deletions ...y/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1279,7 +1279,7 @@
},
"AzureMLLinkedService": {
"x-ms-discriminator-value": "AzureML",
"description": "Azure ML Web Service linked service.",
"description": "Azure ML Studio Web Service linked service.",
"type": "object",
"allOf": [
{
Expand All @@ -1288,7 +1288,7 @@
],
"properties": {
"typeProperties": {
"description": "Azure ML Web Service linked service properties.",
"description": "Azure ML Studio Web Service linked service properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/AzureMLLinkedServiceTypeProperties"
}
Expand All @@ -1298,27 +1298,27 @@
]
},
"AzureMLLinkedServiceTypeProperties": {
"description": "Azure ML Web Service linked service properties.",
"description": "Azure ML Studio Web Service linked service properties.",
"properties": {
"mlEndpoint": {
"type": "object",
"description": "The Batch Execution REST URL for an Azure ML Web Service endpoint. Type: string (or Expression with resultType string)."
"description": "The Batch Execution REST URL for an Azure ML Studio Web Service endpoint. Type: string (or Expression with resultType string)."
},
"apiKey": {
"description": "The API key for accessing the Azure ML model endpoint.",
"$ref": "../datafactory.json#/definitions/SecretBase"
},
"updateResourceEndpoint": {
"type": "object",
"description": "The Update Resource REST URL for an Azure ML Web Service endpoint. Type: string (or Expression with resultType string)."
"description": "The Update Resource REST URL for an Azure ML Studio Web Service endpoint. Type: string (or Expression with resultType string)."
},
"servicePrincipalId": {
"type": "object",
"description": "The ID of the service principal used to authenticate against the ARM-based updateResourceEndpoint of an Azure ML web service. Type: string (or Expression with resultType string)."
"description": "The ID of the service principal used to authenticate against the ARM-based updateResourceEndpoint of an Azure ML Studio web service. Type: string (or Expression with resultType string)."
},
"servicePrincipalKey": {
"$ref": "../datafactory.json#/definitions/SecretBase",
"description": "The key of the service principal used to authenticate against the ARM-based updateResourceEndpoint of an Azure ML web service."
"description": "The key of the service principal used to authenticate against the ARM-based updateResourceEndpoint of an Azure ML Studio web service."
},
"tenant": {
"type": "object",
Expand All @@ -1334,6 +1334,64 @@
"apiKey"
]
},
"AzureMLServiceLinkedService": {
Copy link
Member

@anuchandy anuchandy Oct 9, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@LijuanZ I was trying to find an API endpoint using this model, it seems model is not referenced any where. Is that intentional?

Copy link
Contributor Author

@LijuanZ LijuanZ Oct 9, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@anuchandy This is one new linked service resource type, just similar to all other linked services defined in LinkedService.json file. API endpoint "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.DataFactory/factories/{factoryName}/linkedservices/{linkedServiceName}" references general linkedService type

"x-ms-discriminator-value": "AzureMLService",
"description": "Azure ML Service linked service.",
"type": "object",
"allOf": [
{
"$ref": "#/definitions/LinkedService"
}
],
"properties": {
"typeProperties": {
"description": "Azure ML Service linked service properties.",
"x-ms-client-flatten": true,
"$ref": "#/definitions/AzureMLServiceLinkedServiceTypeProperties"
}
},
"required": [
"typeProperties"
]
},
"AzureMLServiceLinkedServiceTypeProperties": {
"description": "Azure ML Service linked service properties.",
"properties": {
"subscriptionId": {
"type": "object",
"description": "Azure ML Service workspace subscription ID. Type: string (or Expression with resultType string)."
},
"resourceGroupName": {
"type": "object",
"description": "Azure ML Service workspace resource group name. Type: string (or Expression with resultType string)."
},
"mlWorkspaceName": {
"type": "object",
"description": "Azure ML Service workspace name. Type: string (or Expression with resultType string)."
},
"servicePrincipalId": {
"type": "object",
"description": "The ID of the service principal used to authenticate against the endpoint of a published Azure ML Service pipeline. Type: string (or Expression with resultType string)."
},
"servicePrincipalKey": {
"$ref": "../datafactory.json#/definitions/SecretBase",
"description": "The key of the service principal used to authenticate against the endpoint of a published Azure ML Service pipeline."
},
"tenant": {
"type": "object",
"description": "The name or ID of the tenant to which the service principal belongs. Type: string (or Expression with resultType string)."
},
"encryptedCredential": {
"type": "object",
"description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)."
}
},
"required": [
"subscriptionId",
"resourceGroupName",
"mlWorkspaceName"
]
},
"OdbcLinkedService": {
"x-ms-discriminator-value": "Odbc",
"description": "Open Database Connectivity (ODBC) linked service.",
Expand Down
55 changes: 55 additions & 0 deletions ...actory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4956,6 +4956,61 @@
"trainedModelFilePath"
]
},
"AzureMLExecutePipelineActivity": {
"description": "Azure ML Execute Pipeline activity.",
"x-ms-discriminator-value": "AzureMLExecutePipeline",
"allOf": [
{
"$ref": "#/definitions/ExecutionActivity"
}
],
"properties": {
"typeProperties": {
"x-ms-client-flatten": true,
"description": "Azure ML Execute Pipeline activity properties.",
"$ref": "#/definitions/AzureMLExecutePipelineActivityTypeProperties"
}
},
"required": [
"typeProperties"
]
},
"AzureMLExecutePipelineActivityTypeProperties": {
"description": "Azure ML Execute Pipeline activity properties.",
"properties": {
"mlPipelineId": {
"description": "ID of the published Azure ML pipeline. Type: string (or Expression with resultType string).",
"type": "object"
},
"experimentName": {
"description": "Run history experiment name of the pipeline run. This information will be passed in the ExperimentName property of the published pipeline execution request. Type: string (or Expression with resultType string).",
"type": "object"
},
"mlPipelineParameters": {
"description": "Key,Value pairs to be passed to the published Azure ML pipeline endpoint. Keys must match the names of pipeline parameters defined in the published pipeline. Values will be passed in the ParameterAssignments property of the published pipeline execution request. Type: object with key value pairs (or Expression with resultType object).",
"type": "object"
},
"mlParentRunId": {
"description": "The parent Azure ML Service pipeline run id. This information will be passed in the ParentRunId property of the published pipeline execution request. Type: string (or Expression with resultType string).",
"type": "object"
},
"continueOnStepFailure": {
"description": "Whether to continue execution of other steps in the PipelineRun if a step fails. This information will be passed in the continueOnStepFailure property of the published pipeline execution request. Type: boolean (or Expression with resultType boolean).",
"type": "object"
}
},
"required": [
"mlPipelineId"
]
},
"AzureMLPipelineParameters": {
"description": "Key,Value pairs to be passed to the published Azure ML pipeline endpoint. Keys must match the names of pipeline parameters defined in the published pipeline. Values will be passed in the ParameterAssignments property of the published pipeline execution request.",
"type": "object",
"additionalProperties": {
"type": "object",
"description": "Type: string (or Expression with resultType string)."
}
},
"DataLakeAnalyticsUSQLActivity": {
"description": "Data Lake Analytics U-SQL activity.",
"x-ms-discriminator-value": "DataLakeAnalyticsU-SQL",
Expand Down