From a1be5051d2fc99a8ff4bca5c9d27c8f0f9d9851a Mon Sep 17 00:00:00 2001 From: Xiaoning Gao Date: Tue, 14 May 2019 13:31:28 +0800 Subject: [PATCH 1/6] change cluster json file to support the autoscale feature --- .../preview/2015-03-01-preview/cluster.json | 89 +++++++++++++++++++ ...Clusters_UpdateGatewaySettings_Enable.json | 5 ++ .../stable/2018-06-01-preview/cluster.json | 89 +++++++++++++++++++ ...Clusters_UpdateGatewaySettings_Enable.json | 5 ++ 4 files changed, 188 insertions(+) diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json index 263fc59b3384..a86d72f4359f 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json @@ -605,6 +605,90 @@ } } }, + "AutoscaleTimeAndCapacity": { + "properties": { + "time": { + "type": "string", + "description": "24-hour time in the form xx:xx" + }, + "minInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The minimum instance count of the cluster" + }, + "maxInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The maximum instance count of the cluster" + } + }, + "description": "Time and capacity request parameters" + }, + "AutoscaleSchedule": { + "properties": { + "days": { + "type": "array", + "items": { + "type": "string", + "enum": [ "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday" ], + "x-ms-enum": { + "name": "DaysOfWeek", + "modelAsString": false + } + }, + "description": "Days of the week for a schedule-based autoscale rule" + }, + "timeAndCapacity": { + "$ref": "#/definitions/AutoscaleTimeAndCapacity", + "description": "Time and capacity for a schedule-based autoscale rule" + } + }, + "description": "Parameters for a schedule-based autoscale rule, consisting of an array of days + a time and capacity" + }, + "AutoscaleCapacity": { + "properties": { + "minInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The minimum instance count of the cluster" + }, + "maxInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The maximum instance count of the cluster" + } + }, + "description": "The load-based autoscale request parameters" + }, + "AutoscaleRecurrence": { + "properties": { + "timeZone": { + "type": "string", + "description": "The time zone for the autoscale schedule times" + }, + "schedule": { + "type": "array", + "items": { + "$ref": "#/definitions/AutoscaleSchedule" + }, + "description": "Array of schedule-based autoscale rules" + } + }, + "description": "Schedule-based autoscale request parameters" + }, + "Autoscale": { + "properties": { + "capacity": { + "$ref": "#/definitions/AutoscaleCapacity", + "description": "Parameters for load-based autoscale" + }, + "recurrence": { + "$ref": "#/definitions/AutoscaleRecurrence", + "description": "Parameters for schedule-based autoscale" + } + }, + "description": "The autoscale request parameters" + }, "HardwareProfile": { "properties": { "vmSize": { @@ -710,6 +794,11 @@ "format": "int32", "description": "The instance count of the cluster." }, + "autoscale": { + "$ref": "#/definitions/Autoscale", + "x-ms-client-name": "autoscaleConfiguration", + "description": "The autoscale configurations." + }, "hardwareProfile": { "$ref": "#/definitions/HardwareProfile", "description": "The hardware profile." diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json index da62bfa19376..d29d32c8b776 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json @@ -15,6 +15,11 @@ "headers": { "location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult" } + }, + "200": { + "headers": { + "location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult" + } } } } diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json index fc18241a07fa..13a819d20bf4 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json @@ -605,6 +605,90 @@ } } }, + "AutoscaleTimeAndCapacity": { + "properties": { + "time": { + "type": "string", + "description": "24-hour time in the form xx:xx" + }, + "minInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The minimum instance count of the cluster" + }, + "maxInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The maximum instance count of the cluster" + } + }, + "description": "Time and capacity request parameters" + }, + "AutoscaleSchedule": { + "properties": { + "days": { + "type": "array", + "items": { + "type": "string", + "enum": [ "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday" ], + "x-ms-enum": { + "name": "DaysOfWeek", + "modelAsString": false + } + }, + "description": "Days of the week for a schedule-based autoscale rule" + }, + "timeAndCapacity": { + "$ref": "#/definitions/AutoscaleTimeAndCapacity", + "description": "Time and capacity for a schedule-based autoscale rule" + } + }, + "description": "Parameters for a schedule-based autoscale rule, consisting of an array of days + a time and capacity" + }, + "AutoscaleCapacity": { + "properties": { + "minInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The minimum instance count of the cluster" + }, + "maxInstanceCount": { + "type": "integer", + "format": "int32", + "description": "The maximum instance count of the cluster" + } + }, + "description": "The load-based autoscale request parameters" + }, + "AutoscaleRecurrence": { + "properties": { + "timeZone": { + "type": "string", + "description": "The time zone for the autoscale schedule times" + }, + "schedule": { + "type": "array", + "items": { + "$ref": "#/definitions/AutoscaleSchedule" + }, + "description": "Array of schedule-based autoscale rules" + } + }, + "description": "Schedule-based autoscale request parameters" + }, + "Autoscale": { + "properties": { + "capacity": { + "$ref": "#/definitions/AutoscaleCapacity", + "description": "Parameters for load-based autoscale" + }, + "recurrence": { + "$ref": "#/definitions/AutoscaleRecurrence", + "description": "Parameters for schedule-based autoscale" + } + }, + "description": "The autoscale request parameters" + }, "HardwareProfile": { "properties": { "vmSize": { @@ -710,6 +794,11 @@ "format": "int32", "description": "The instance count of the cluster." }, + "autoscale": { + "$ref": "#/definitions/Autoscale", + "x-ms-client-name": "autoscaleConfiguration", + "description": "The autoscale configurations." + }, "hardwareProfile": { "$ref": "#/definitions/HardwareProfile", "description": "The hardware profile." diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json index 306db252822e..23c05b00feca 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/HDI_Clusters_UpdateGatewaySettings_Enable.json @@ -15,6 +15,11 @@ "headers": { "location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult" } + }, + "200": { + "headers": { + "location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult" + } } } } From 01e68266bd5489f236adce0613bdfcefa577c839 Mon Sep 17 00:00:00 2001 From: Xiaoning Gao Date: Tue, 18 Jun 2019 19:30:39 +0800 Subject: [PATCH 2/6] add x-ms-examples for Autoscale --- .../preview/2015-03-01-preview/cluster.json | 3 +++ .../Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json index a86d72f4359f..ed54bb3b793d 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json @@ -56,6 +56,9 @@ }, "Create Hadoop cluster with Azure Data Lake Storage Gen 2": { "$ref": "./examples/CreateLinuxHadoopAdlsGen2.json" + }, + "Create HDInsight cluster with Autoscale configuration": { + "$ref": "./examples/CreateHDInsightClusterWithAutoscaleConfig.json" } }, "parameters": [ diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json index 13a819d20bf4..853c0ca854db 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/cluster.json @@ -56,6 +56,9 @@ }, "Create Hadoop cluster with Azure Data Lake Storage Gen 2": { "$ref": "./examples/CreateLinuxHadoopAdlsGen2.json" + }, + "Create HDInsight cluster with Autoscale configuration": { + "$ref": "./examples/CreateHDInsightClusterWithAutoscaleConfig.json" } }, "parameters": [ From 3b14442f6c54c0afa220c129b4e14346dd59e9b5 Mon Sep 17 00:00:00 2001 From: Xiaoning Gao Date: Tue, 18 Jun 2019 19:31:28 +0800 Subject: [PATCH 3/6] add x-ms-examples for Autoscale --- ...teHDInsightClusterWithAutoscaleConfig.json | 125 ++++++++++++++++++ ...teHDInsightClusterWithAutoscaleConfig.json | 125 ++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json create mode 100644 specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json new file mode 100644 index 000000000000..08a2e7fb4c69 --- /dev/null +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json @@ -0,0 +1,125 @@ +{ + "parameters": { + "clusterName": "cluster1", + "resourceGroupName": "rg1", + "api-version": "2015-03-01-preview", + "subscriptionId": "subid", + "name": "HDinsight-autoscale-example", + "type": "Microsoft.HDInsight/clusters", + "parameters": { + "properties": { + "clusterVersion": "3.6", + "osType": "Linux", + "tier": "standard", + "clusterDefinition": { + "kind": "hadoop", + "componentVersion": { + "Hadoop": "2.7" + }, + "configurations": { + "gateway": { + "restAuthCredential.isEnabled": true, + "restAuthCredential.username": "admin", + "restAuthCredential.password": "**********" + } + } + }, + "storageProfile": { + "storageaccounts": [ + { + "name": "mystorage.blob.core.windows.net", + "isDefault": true, + "container": "hdinsight-autoscale-tes-2019-06-18t05-49-16-591z", + "key": "storagekey" + } + ] + }, + "computeProfile": { + "roles": [ + { + "autoscale": { + "capacity": null, + "recurrence": { + "timeZone": "China Standard Time", + "schedule": [ + { + "days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "timeAndCapacity": { + "time": "09:00", + "minInstanceCount": 3, + "maxInstanceCount": 3 + } + }, + { + "days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "timeAndCapacity": { + "time": "18:00", + "minInstanceCount": 6, + "maxInstanceCount": 6 + } + }, + { + "days": [ + "Saturday", + "Sunday" + ], + "timeAndCapacity": { + "time": "09:00", + "minInstanceCount": 2, + "maxInstanceCount": 2 + } + }, + { + "days": [ + "Saturday", + "Sunday" + ], + "timeAndCapacity": { + "time": "18:00", + "minInstanceCount": 4, + "maxInstanceCount": 4 + } + } + ] + } + }, + "name": "workernode", + "targetInstanceCount": 4, + "hardwareProfile": { + "vmSize": "Standard_D4_V2" + }, + "osProfile": { + "linuxOperatingSystemProfile": { + "username": "sshuser", + "password": "**********" + } + }, + "virtualNetworkProfile": null, + "scriptActions": [], + "dataDisksGroups": null + } + ] + } + } + } + }, + "responses": { + "200": { + "headers": { + "location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult" + } + } + } +} \ No newline at end of file diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json new file mode 100644 index 000000000000..28ba2dcc02db --- /dev/null +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json @@ -0,0 +1,125 @@ +{ + "parameters": { + "clusterName": "cluster1", + "resourceGroupName": "rg1", + "api-version": "2018-06-01-preview", + "subscriptionId": "subid", + "name": "HDinsight-autoscale-example", + "type": "Microsoft.HDInsight/clusters", + "parameters": { + "properties": { + "clusterVersion": "3.6", + "osType": "Linux", + "tier": "standard", + "clusterDefinition": { + "kind": "hadoop", + "componentVersion": { + "Hadoop": "2.7" + }, + "configurations": { + "gateway": { + "restAuthCredential.isEnabled": true, + "restAuthCredential.username": "admin", + "restAuthCredential.password": "**********" + } + } + }, + "storageProfile": { + "storageaccounts": [ + { + "name": "mystorage.blob.core.windows.net", + "isDefault": true, + "container": "hdinsight-autoscale-tes-2019-06-18t05-49-16-591z", + "key": "storagekey" + } + ] + }, + "computeProfile": { + "roles": [ + { + "autoscale": { + "capacity": null, + "recurrence": { + "timeZone": "China Standard Time", + "schedule": [ + { + "days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "timeAndCapacity": { + "time": "09:00", + "minInstanceCount": 3, + "maxInstanceCount": 3 + } + }, + { + "days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "timeAndCapacity": { + "time": "18:00", + "minInstanceCount": 6, + "maxInstanceCount": 6 + } + }, + { + "days": [ + "Saturday", + "Sunday" + ], + "timeAndCapacity": { + "time": "09:00", + "minInstanceCount": 2, + "maxInstanceCount": 2 + } + }, + { + "days": [ + "Saturday", + "Sunday" + ], + "timeAndCapacity": { + "time": "18:00", + "minInstanceCount": 4, + "maxInstanceCount": 4 + } + } + ] + } + }, + "name": "workernode", + "targetInstanceCount": 4, + "hardwareProfile": { + "vmSize": "Standard_D4_V2" + }, + "osProfile": { + "linuxOperatingSystemProfile": { + "username": "sshuser", + "password": "**********" + } + }, + "virtualNetworkProfile": null, + "scriptActions": [], + "dataDisksGroups": null + } + ] + } + } + } + }, + "responses": { + "200": { + "headers": { + "location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult" + } + } + } +} \ No newline at end of file From 991b04c7183d470c85fef6402e487f8974e49220 Mon Sep 17 00:00:00 2001 From: Xiaoning Gao Date: Wed, 19 Jun 2019 16:44:34 +0800 Subject: [PATCH 4/6] fix valid error --- .../examples/CreateHDInsightClusterWithAutoscaleConfig.json | 2 -- .../examples/CreateHDInsightClusterWithAutoscaleConfig.json | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json index 08a2e7fb4c69..2304b9f6e3d5 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json @@ -4,8 +4,6 @@ "resourceGroupName": "rg1", "api-version": "2015-03-01-preview", "subscriptionId": "subid", - "name": "HDinsight-autoscale-example", - "type": "Microsoft.HDInsight/clusters", "parameters": { "properties": { "clusterVersion": "3.6", diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json index 28ba2dcc02db..043f22bf652d 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json @@ -4,10 +4,11 @@ "resourceGroupName": "rg1", "api-version": "2018-06-01-preview", "subscriptionId": "subid", - "name": "HDinsight-autoscale-example", - "type": "Microsoft.HDInsight/clusters", "parameters": { "properties": { + "tags": { + "key1": "val1" + }, "clusterVersion": "3.6", "osType": "Linux", "tier": "standard", From dd08127df872a86b9d3549d65a68dfb1b70ae878 Mon Sep 17 00:00:00 2001 From: Xiaoning Gao Date: Wed, 19 Jun 2019 17:03:47 +0800 Subject: [PATCH 5/6] fix valid error --- .../examples/CreateHDInsightClusterWithAutoscaleConfig.json | 2 +- .../examples/CreateHDInsightClusterWithAutoscaleConfig.json | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json index 2304b9f6e3d5..bd255b6f85cb 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json @@ -8,7 +8,7 @@ "properties": { "clusterVersion": "3.6", "osType": "Linux", - "tier": "standard", + "tier": "Standard", "clusterDefinition": { "kind": "hadoop", "componentVersion": { diff --git a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json index 043f22bf652d..19a92e1154e3 100644 --- a/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json +++ b/specification/hdinsight/resource-manager/Microsoft.HDInsight/stable/2018-06-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json @@ -6,12 +6,9 @@ "subscriptionId": "subid", "parameters": { "properties": { - "tags": { - "key1": "val1" - }, "clusterVersion": "3.6", "osType": "Linux", - "tier": "standard", + "tier": "Standard", "clusterDefinition": { "kind": "hadoop", "componentVersion": { From edd2f703e47a925dbfa1feaa75907dacbdd3d3fe Mon Sep 17 00:00:00 2001 From: Xiaoning Gao Date: Fri, 21 Jun 2019 10:51:38 +0800 Subject: [PATCH 6/6] Trigger notification