diff --git a/05-bootstrap-prep.md b/05-bootstrap-prep.md index 5c9f86e8..ff819a67 100644 --- a/05-bootstrap-prep.md +++ b/05-bootstrap-prep.md @@ -42,7 +42,7 @@ We'll be bootstrapping this cluster with the Flux GitOps agent as installed as a ```bash # [This takes about four minutes.] - az deployment group create -g rg-bu0001a0008 -f acr-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} + az deployment group create -g rg-bu0001a0008 -f acr-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} location=eastus2 ``` 1. Import cluster management images to your container registry. @@ -55,7 +55,7 @@ We'll be bootstrapping this cluster with the Flux GitOps agent as installed as a echo ACR_NAME_AKS_BASELINE: $ACR_NAME_AKS_BASELINE # Import core image(s) hosted in public container registries to be used during bootstrapping - az acr import --source docker.io/weaveworks/kured:1.9.0 -n $ACR_NAME_AKS_BASELINE + az acr import --source docker.io/weaveworks/kured:1.9.2 -n $ACR_NAME_AKS_BASELINE ``` > In this walkthrough, there is only one image that is included in the bootstrapping process. It's included as an reference for this process. Your choice to use Kubernetes Reboot Daemon (Kured) or any other images, including helm charts, as part of your bootstrapping is yours to make. diff --git a/06-aks-cluster.md b/06-aks-cluster.md index 2ebee237..7e667083 100644 --- a/06-aks-cluster.md +++ b/06-aks-cluster.md @@ -23,7 +23,7 @@ Now that your [ACR instance is deployed and ready to support cluster bootstrappi ```bash # [This takes about 18 minutes.] - az deployment group create -g rg-bu0001a0008 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE} a0008NamespaceReaderAadGroupObjectId=${AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC_AKS_BASELINE} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE} domainName=${DOMAIN_NAME_AKS_BASELINE} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME} + az deployment group create -g rg-bu0001a0008 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE_AKS_BASELINE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN_AKS_BASELINE} a0008NamespaceReaderAadGroupObjectId=${AADOBJECTID_GROUP_A0008_READER_AKS_BASELINE} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC_AKS_BASELINE} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_AKS_BASELINE} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64_AKS_BASELINE} domainName=${DOMAIN_NAME_AKS_BASELINE} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME} location=eastus2 ``` > Alteratively, you could have updated the [`azuredeploy.parameters.prod.json`](./azuredeploy.parameters.prod.json) file and deployed as above, using `-p "@azuredeploy.parameters.prod.json"` instead of providing the individual key-value pairs. diff --git a/README.md b/README.md index 822c8f43..015143e9 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Finally, this implementation uses the [ASP.NET Core Docker sample web app](https #### Azure platform -- AKS v1.22 +- AKS v1.23 - System and User [node pool separation](https://docs.microsoft.com/azure/aks/use-system-pools) - [AKS-managed Azure AD](https://docs.microsoft.com/azure/aks/managed-aad) - Azure AD-backed Kubernetes RBAC (_local user accounts disabled_) diff --git a/cluster-manifests/cluster-baseline-settings/kured.yaml b/cluster-manifests/cluster-baseline-settings/kured.yaml index 533d2333..90aac334 100644 --- a/cluster-manifests/cluster-baseline-settings/kured.yaml +++ b/cluster-manifests/cluster-baseline-settings/kured.yaml @@ -1,4 +1,4 @@ -# Source: https://github.com/weaveworks/kured/releases/download/1.9.0/kured-1.9.0-dockerhub.yaml +# Source: https://github.com/weaveworks/kured/releases/download/1.9.2/kured-1.9.2-dockerhub.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -106,10 +106,10 @@ spec: # PRODUCTION READINESS CHANGE REQUIRED # This image should be sourced from a non-public container registry, such as the # one deployed along side of this reference implementation. - # az acr import --source docker.io/weaveworks/kured:1.9.0 -n + # az acr import --source docker.io/weaveworks/kured:1.9.2 -n # and then set this to - # image: .azurecr.io/weaveworks/kured:1.9.0 - image: docker.io/weaveworks/kured:1.9.0 + # image: .azurecr.io/weaveworks/kured:1.9.2 + image: docker.io/weaveworks/kured:1.9.2 imagePullPolicy: IfNotPresent resources: limits: diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 0d5627a2..91491b3b 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -43,7 +43,7 @@ param clusterAuthorizedIPRanges array = [] 'southeastasia' ]) param location string = 'eastus2' -param kubernetesVersion string = '1.22.4' +param kubernetesVersion string = '1.23.5' @description('Domain name to use for App Gateway and AKS ingress.') param domainName string = 'contoso.com' @@ -58,11 +58,6 @@ param gitOpsBootstrappingRepoBranch string = 'main' /*** VARIABLES ***/ -var monitoringMetricsPublisherRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/3913510d-42f4-4e42-8a64-420c390055eb' -var acrPullRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d' -var managedIdentityOperatorRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/f1a07417-d97a-45cb-824c-7a7467783830' -var keyVaultReader = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/21090545-7ca7-4776-b22c-e363652d74d2' -var keyVaultSecretsUserRole = '${subscription().id}/providers/Microsoft.Authorization/roleDefinitions/4633458b-17de-408a-b874-0445c86b69e6' var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' @@ -109,6 +104,36 @@ resource clusterReaderRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-p scope: subscription() } +// Built-in Azure RBAC role that is applied to a cluster to grant its monitoring agent's identity with publishing metrics and push alerts permissions. +resource monitoringMetricsPublisherRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { + name: '3913510d-42f4-4e42-8a64-420c390055eb' + scope: subscription() +} + +// Built-in Azure RBAC role that can be applied to an Azure Container Registry to grant the authority pull container images. Granted to the AKS cluster's kubelet identity. +resource acrPullRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { + name: '7f951dda-4ed3-4680-a7ca-43fe172d538d' + scope: subscription() +} + +// Built-in Azure RBAC role that must be applied to the kublet Managed Identity allowing it to further assign adding managed identities to the cluster's underlying VMSS. +resource managedIdentityOperatorRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { + name: 'f1a07417-d97a-45cb-824c-7a7467783830' + scope: subscription() +} + +// Built-in Azure RBAC role that is applied a Key Vault to grant with metadata, certificates, keys and secrets read privileges. Granted to App Gateway's managed identity. +resource keyVaultReaderRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { + name: '21090545-7ca7-4776-b22c-e363652d74d2' + scope: subscription() +} + +// Built-in Azure RBAC role that is applied to a Key Vault to grant with secrets content read privileges. Granted to both Key Vault and our workload's identity. +resource keyVaultSecretsUserRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { + name: '4633458b-17de-408a-b874-0445c86b69e6' + scope: subscription() +} + /*** EXISTING HUB RESOURCES ***/ resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = { @@ -1109,9 +1134,9 @@ resource kv_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-0 // Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole) + name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultSecretsUserRole.id) properties: { - roleDefinitionId: keyVaultSecretsUserRole + roleDefinitionId: keyVaultSecretsUserRole.id principalId: miAppGatewayFrontend.properties.principalId principalType: 'ServicePrincipal' } @@ -1121,9 +1146,9 @@ resource kvMiAppGatewayFrontendSecretsUserRole_roleAssignment 'Microsoft.Authori // Grant the Azure Application Gateway managed identity with key vault reader role permissions; this allows pulling frontend and backend certificates. resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReader) + name: guid(resourceGroup().id, 'mi-appgateway-frontend', keyVaultReaderRole.id) properties: { - roleDefinitionId: keyVaultReader + roleDefinitionId: keyVaultReaderRole.id principalId: miAppGatewayFrontend.properties.principalId principalType: 'ServicePrincipal' } @@ -1133,9 +1158,9 @@ resource kvMiAppGatewayFrontendKeyVaultReader_roleAssignment 'Microsoft.Authoriz // Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates. resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole) + name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultSecretsUserRole.id) properties: { - roleDefinitionId: keyVaultSecretsUserRole + roleDefinitionId: keyVaultSecretsUserRole.id principalId: podmiIngressController.properties.principalId principalType: 'ServicePrincipal' } @@ -1145,9 +1170,9 @@ resource kvPodMiIngressControllerSecretsUserRole_roleAssignment 'Microsoft.Autho // Grant the AKS cluster ingress controller pod managed identity with key vault reader role permissions; this allows our ingress controller to pull certificates resource kvPodMiIngressControllerKeyVaultReader_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: kv - name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReader) + name: guid(resourceGroup().id, 'podmi-ingress-controller', keyVaultReaderRole.id) properties: { - roleDefinitionId: keyVaultReader + roleDefinitionId: keyVaultReaderRole.id principalId: podmiIngressController.properties.principalId principalType: 'ServicePrincipal' } @@ -1437,9 +1462,9 @@ resource mc 'Microsoft.ContainerService/managedClusters@2022-01-02-preview' = { resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: acr - name: guid(mc.id, acrPullRole) + name: guid(mc.id, acrPullRole.id) properties: { - roleDefinitionId: acrPullRole + roleDefinitionId: acrPullRole.id description: 'Allows AKS to pull container images from this ACR instance.' principalId: mc.properties.identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' @@ -1450,9 +1475,9 @@ resource acrKubeletAcrPullRole_roleAssignment 'Microsoft.Authorization/roleAssig // Grant the OMS Agent's Managed Identity the metrics publisher role to push alerts resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: mc - name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole) + name: guid(mc.id, 'omsagent', monitoringMetricsPublisherRole.id) properties: { - roleDefinitionId: monitoringMetricsPublisherRole + roleDefinitionId: monitoringMetricsPublisherRole.id principalId: mc.properties.addonProfiles.omsagent.identity.objectId principalType: 'ServicePrincipal' } @@ -1462,9 +1487,9 @@ resource mcOmsAgentMonitoringMetricsPublisherRole_roleAssignment 'Microsoft.Auth // Grant the AKS cluster with Managed Identity Operator role permissions over the managed identity used for the ingress controller. Allows it to be assigned to the underlying VMSS. resource miKubeletManagedIdentityOperatorRole_roleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = { scope: podmiIngressController - name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole) + name: guid(resourceGroup().id, 'podmi-ingress-controller', managedIdentityOperatorRole.id) properties: { - roleDefinitionId: managedIdentityOperatorRole + roleDefinitionId: managedIdentityOperatorRole.id principalId: mc.properties.identityProfile.kubeletidentity.objectId principalType: 'ServicePrincipal' } @@ -1855,5 +1880,6 @@ resource agwdiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01 output aksClusterName string = clusterName output aksIngressControllerPodManagedIdentityResourceId string = podmiIngressController.id -output aksIngressControllerPodManagedIdentityClientId string = reference(podmiIngressController.id, '2018-11-30').clientId +output aksIngressControllerPodManagedIdentityClientId string = podmiIngressController.properties.clientId +output aksOidcIssuerUrl string = mc.properties.oidcIssuerProfile.issuerURL output keyVaultName string = kv.name