azurerm_kubernetes_cluster: Support for BYO kubelet_identity

azurerm/internal/services/containers/kubernetes_cluster_auth_resource_test.go

@@ -60,11 +60,6 @@ func TestAccKubernetesCluster_managedClusterIdentity(t *testing.T) {
 	testAccKubernetesCluster_managedClusterIdentity(t)
 }
 
-func TestAccKubernetesCluster_userAssignedIdentity(t *testing.T) {
-	checkIfShouldRunTestsIndividually(t)
-	testAccKubernetesCluster_userAssignedIdentity(t)
-}
-
 func testAccKubernetesCluster_managedClusterIdentity(t *testing.T) {
 	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
 	r := KubernetesClusterResource{}
@@ -85,6 +80,11 @@ func testAccKubernetesCluster_managedClusterIdentity(t *testing.T) {
 	})
 }
 
+func TestAccKubernetesCluster_userAssignedIdentity(t *testing.T) {
+	checkIfShouldRunTestsIndividually(t)
+	testAccKubernetesCluster_userAssignedIdentity(t)
+}
+
 func testAccKubernetesCluster_userAssignedIdentity(t *testing.T) {
 	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
 	r := KubernetesClusterResource{}
@@ -129,6 +129,29 @@ func testAccKubernetesCluster_updateWithUserAssignedIdentity(t *testing.T) {
 	})
 }
 
+func TestAccKubernetesCluster_userAssignedKubeletIdentity(t *testing.T) {
+	checkIfShouldRunTestsIndividually(t)
+	testAccKubernetesCluster_userAssignedKubeletIdentity(t)
+}
+
+func testAccKubernetesCluster_userAssignedKubeletIdentity(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
+	r := KubernetesClusterResource{}
+
+	data.ResourceTest(t, r, []acceptance.TestStep{
+		{
+			Config: r.userAssignedKubeletIdentityConfig(data),
+			Check: acceptance.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("identity.0.type").HasValue("UserAssigned"),
+				check.That(data.ResourceName).Key("identity.0.user_assigned_identity_id").Exists(),
+				check.That(data.ResourceName).Key("kubelet_identity.0.user_assigned_identity_id").Exists(),
+			),
+		},
+		data.ImportStep(),
+	})
+}
+
 func TestAccKubernetesCluster_roleBasedAccessControl(t *testing.T) {
 	checkIfShouldRunTestsIndividually(t)
 	testAccKubernetesCluster_roleBasedAccessControl(t)
@@ -616,6 +639,63 @@ resource "azurerm_kubernetes_cluster" "test" {
 `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger)
 }
 
+func (KubernetesClusterResource) userAssignedKubeletIdentityConfig(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+provider "azurerm" {
+  features {}
+}
+
+resource "azurerm_resource_group" "test" {
+  name     = "acctestRG-aks-%d"
+  location = "%s"
+}
+
+resource "azurerm_user_assigned_identity" "aks_identity_test" {
+  resource_group_name = azurerm_resource_group.test.name
+  location            = azurerm_resource_group.test.location
+  name                = "test_identity"
+}
+
+resource "azurerm_user_assigned_identity" "kubelet_identity_test" {
+  resource_group_name = azurerm_resource_group.test.name
+  location            = azurerm_resource_group.test.location
+  name                = "test_kubelet_identity"
+}
+
+resource "azurerm_role_assignment" "manage_kubelet_identity" {
+  scope                            = azurerm_resource_group.test.id
+  role_definition_name             = "Managed Identity Operator"
+  principal_id                     = azurerm_user_assigned_identity.aks_identity_test.principal_id
+  skip_service_principal_aad_check = false
+}
+
+resource "azurerm_kubernetes_cluster" "test" {
+  depends_on          = [azurerm_role_assignment.manage_kubelet_identity]
+  name                = "acctestaks%d"
+  location            = azurerm_resource_group.test.location
+  resource_group_name = azurerm_resource_group.test.name
+  dns_prefix          = "acctestaks%d"
+
+  default_node_pool {
+    name       = "default"
+    node_count = 1
+    vm_size    = "Standard_DS2_v2"
+  }
+
+  identity {
+    type                      = "UserAssigned"
+    user_assigned_identity_id = azurerm_user_assigned_identity.aks_identity_test.id
+  }
+
+  kubelet_identity {
+    user_assigned_identity_id = azurerm_user_assigned_identity.kubelet_identity_test.id
+    client_id                 = azurerm_user_assigned_identity.kubelet_identity_test.client_id
+    object_id                 = azurerm_user_assigned_identity.kubelet_identity_test.principal_id
+  }
+}
+`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger)
+}
+
 func (KubernetesClusterResource) roleBasedAccessControlConfig(data acceptance.TestData) string {
 	return fmt.Sprintf(`
 provider "azurerm" {

azurerm/internal/services/containers/kubernetes_cluster_resource.go

@@ -262,19 +262,33 @@ func resourceKubernetesCluster() *pluginsdk.Resource {
 			"kubelet_identity": {
 				Type:     pluginsdk.TypeList,
 				Computed: true,
+				Optional: true,
+				MaxItems: 1,
 				Elem: &pluginsdk.Resource{
 					Schema: map[string]*pluginsdk.Schema{
 						"client_id": {
-							Type:     pluginsdk.TypeString,
-							Computed: true,
+							Type:         pluginsdk.TypeString,
+							Optional:     true,
+							Computed:     true,
+							ForceNew:     true,
+							RequiredWith: []string{"kubelet_identity.0.object_id", "kubelet_identity.0.user_assigned_identity_id", "identity.0.user_assigned_identity_id"},
+							ValidateFunc: validation.StringIsNotEmpty,
 						},
 						"object_id": {
-							Type:     pluginsdk.TypeString,
-							Computed: true,
+							Type:         pluginsdk.TypeString,
+							Optional:     true,
+							Computed:     true,
+							ForceNew:     true,
+							RequiredWith: []string{"kubelet_identity.0.client_id", "kubelet_identity.0.user_assigned_identity_id", "identity.0.user_assigned_identity_id"},
+							ValidateFunc: validation.StringIsNotEmpty,
 						},
 						"user_assigned_identity_id": {
-							Type:     pluginsdk.TypeString,
-							Computed: true,
+							Type:         pluginsdk.TypeString,
+							Optional:     true,
+							Computed:     true,
+							ForceNew:     true,
+							RequiredWith: []string{"kubelet_identity.0.client_id", "kubelet_identity.0.object_id", "identity.0.user_assigned_identity_id"},
+							ValidateFunc: msivalidate.UserAssignedIdentityID,
 						},
 					},
 				},
@@ -907,6 +921,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
 	}
 
 	managedClusterIdentityRaw := d.Get("identity").([]interface{})
+	kubernetesClusterIdentityRaw := d.Get("kubelet_identity").([]interface{})
 	servicePrincipalProfileRaw := d.Get("service_principal").([]interface{})
 
 	if len(managedClusterIdentityRaw) == 0 && len(servicePrincipalProfileRaw) == 0 {
@@ -919,6 +934,9 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{}
 			ClientID: utils.String("msi"),
 		}
 	}
+	if len(kubernetesClusterIdentityRaw) > 0 {
+		parameters.ManagedClusterProperties.IdentityProfile = expandKubernetesClusterIdentityProfile(kubernetesClusterIdentityRaw)
+	}
 
 	servicePrincipalSet := false
 	if len(servicePrincipalProfileRaw) > 0 {
@@ -1543,6 +1561,25 @@ func expandKubernetesClusterLinuxProfile(input []interface{}) *containerservice.
 	}
 }
 
+func expandKubernetesClusterIdentityProfile(input []interface{}) map[string]*containerservice.ManagedClusterPropertiesIdentityProfileValue {
+	identityProfile := make(map[string]*containerservice.ManagedClusterPropertiesIdentityProfileValue)
+	if len(input) == 0 || input[0] == nil {
+		return identityProfile
+	}
+
+	values := input[0].(map[string]interface{})
+
+	if containerservice.ResourceIdentityType(values["user_assigned_identity_id"].(string)) != "" {
+		identityProfile["kubeletidentity"] = &containerservice.ManagedClusterPropertiesIdentityProfileValue{
+			ResourceID: utils.String(values["user_assigned_identity_id"].(string)),
+			ClientID:   utils.String(values["client_id"].(string)),
+			ObjectID:   utils.String(values["object_id"].(string)),
+		}
+	}
+
+	return identityProfile
+}
+
 func flattenKubernetesClusterIdentityProfile(profile map[string]*containerservice.ManagedClusterPropertiesIdentityProfileValue) ([]interface{}, error) {
 	if profile == nil {
 		return []interface{}{}, nil

website/docs/r/kubernetes_cluster.html.markdown

@@ -96,6 +96,8 @@ In addition, one of either `identity` or `service_principal` blocks must be spec
 
 -> **NOTE:** One of either `identity` or `service_principal` must be specified.
 
+* `kubelet_identity` - A `kubelet_identity` block as defined below. Changing this forces a new resource to be created.
+
 * `kubernetes_version` - (Optional) Version of Kubernetes specified when creating the AKS managed cluster. If not specified, the latest recommended version will be used at provisioning time (but won't auto-upgrade).
 
 -> **NOTE:** Upgrading your cluster may take up to 10 minutes per node.
@@ -366,6 +368,17 @@ An `identity` block supports the following:
 
 ---
 
+The `kubelet_identity` block supports the following:
+
+* `client_id` - (Required) The Client ID of the user-defined Managed Identity to be assigned to the Kubelets. If not specified a Managed Identity is created automatically.
+
+* `object_id` - (Required) The Object ID of the user-defined Managed Identity assigned to the Kubelets.If not specified a Managed Identity is created automatically.
+
+* `user_assigned_identity_id` - (Required) The ID of the User Assigned Identity assigned to the Kubelets. If not specified a Managed Identity is created automatically. 
+
+-> **NOTE:** The functionality to bring your own `kubelet_identity` is in Public Preview, and therefore has some limitations - please [see the Azure documentation for more information](https://docs.microsoft.com/en-us/azure/aks/use-managed-identity#limitations-2). It requires a BYO Cluster Identity  `identity.0.user_assigned_identity_id`) to be specified.
+---
+
 A `kube_dashboard` block supports the following:
 
 * `enabled` - (Required) Is the Kubernetes Dashboard enabled?
@@ -518,9 +531,7 @@ The following attributes are exported:
 
 * `http_application_routing` - A `http_application_routing` block as defined below.
 
-* `node_resource_group` - The auto-generated Resource Group which contains the resources for this Managed Kubernetes Cluster.
-
-* `kubelet_identity` - A `kubelet_identity` block as defined below.  
+* `node_resource_group` - The auto-generated Resource Group which contains the resources for this Managed Kubernetes Cluster. 
 
 * `addon_profile` - An `addon_profile` block as defined below.
 
@@ -546,16 +557,6 @@ The `identity` block exports the following:
 
 ---
 
-The `kubelet_identity` block exports the following:
-
-* `client_id` - The Client ID of the user-defined Managed Identity assigned to the Kubelets.
-
-* `object_id` - The Object ID of the user-defined Managed Identity assigned to the Kubelets.
-
-* `user_assigned_identity_id` - The ID of the User Assigned Identity assigned to the Kubelets.
-
----
-
 The `kube_admin_config` and `kube_config` blocks export the following:
 
 * `client_key` - Base64 encoded private key used by clients to authenticate to the Kubernetes cluster.