Bug 1788707: Backport Azure private DNS to 4.2 branch

data/data/azure/bootstrap/variables.tf

@@ -59,11 +59,6 @@ variable "tags" {
   description = "tags to be applied to created resources."
 }
 
-variable "private_dns_zone_id" {
-  type        = string
-  description = "This is to create explicit dependency on private zone to exist before VMs are created in the vnet. https://github.com/MicrosoftDocs/azure-docs/issues/13728"
-}
-
 variable "nsg_name" {
   type        = string
   description = "The network security group for the subnet."

data/data/azure/dns/dns.tf

@@ -3,17 +3,29 @@ locals {
   api_external_name = "api.${replace(var.cluster_domain, ".${var.base_domain}", "")}"
 }
 
-resource "azurerm_dns_a_record" "apiint_internal" {
+resource "azureprivatedns_zone" "private" {
+  name                = var.cluster_domain
+  resource_group_name = var.resource_group_name
+}
+
+resource "azureprivatedns_zone_virtual_network_link" "network" {
+  name                  = "${var.cluster_id}-network-link"
+  resource_group_name   = var.resource_group_name
+  private_dns_zone_name = azureprivatedns_zone.private.name
+  virtual_network_id    = var.virtual_network
+}
+
+resource "azureprivatedns_a_record" "apiint_internal" {
   name                = "api-int"
-  zone_name           = var.private_dns_zone_name
+  zone_name           = azureprivatedns_zone.private.name
   resource_group_name = var.resource_group_name
   ttl                 = 300
   records             = [var.internal_lb_ipaddress]
 }
 
-resource "azurerm_dns_a_record" "api_internal" {
+resource "azureprivatedns_a_record" "api_internal" {
   name                = "api"
-  zone_name           = var.private_dns_zone_name
+  zone_name           = azureprivatedns_zone.private.name
   resource_group_name = var.resource_group_name
   ttl                 = 300
   records             = [var.internal_lb_ipaddress]
@@ -27,26 +39,26 @@ resource "azurerm_dns_cname_record" "api_external" {
   record              = var.external_lb_fqdn
 }
 
-resource "azurerm_dns_a_record" "etcd_a_nodes" {
+resource "azureprivatedns_a_record" "etcd_a_nodes" {
   count               = var.etcd_count
   name                = "etcd-${count.index}"
-  zone_name           = var.private_dns_zone_name
+  zone_name           = azureprivatedns_zone.private.name
   resource_group_name = var.resource_group_name
   ttl                 = 60
   records             = [var.etcd_ip_addresses[count.index]]
 }
 
-resource "azurerm_dns_srv_record" "etcd_cluster" {
+resource "azureprivatedns_srv_record" "etcd_cluster" {
   name                = "_etcd-server-ssl._tcp"
-  zone_name           = var.private_dns_zone_name
+  zone_name           = azureprivatedns_zone.private.name
   resource_group_name = var.resource_group_name
   ttl                 = 60
 
   dynamic "record" {
-    for_each = azurerm_dns_a_record.etcd_a_nodes.*.name
+    for_each = azureprivatedns_a_record.etcd_a_nodes.*.name
     iterator = name
     content {
-      target   = "${name.value}.${var.private_dns_zone_name}"
+      target   = "${name.value}.${azureprivatedns_zone.private.name}"
       priority = 10
       weight   = 10
       port     = 2380

data/data/azure/dns/variables.tf

@@ -4,6 +4,11 @@ variable "tags" {
   description = "tags to be applied to created resources."
 }
 
+variable "cluster_id" {
+  description = "The identifier for the cluster."
+  type        = string
+}
+
 variable "cluster_domain" {
   description = "The domain for the cluster that all DNS records must belong"
   type        = string
@@ -29,8 +34,8 @@ variable "internal_lb_ipaddress" {
   type        = string
 }
 
-variable "private_dns_zone_name" {
-  description = "private DNS zone name that should be used for records"
+variable "virtual_network" {
+  description = "The ID for Virtual Network that will be linked to the Private DNS zone."
   type        = string
 }
 

data/data/azure/main.tf

@@ -5,9 +5,6 @@ locals {
     },
     var.azure_extra_tags,
   )
-
-  master_subnet_cidr = cidrsubnet(var.machine_cidr, 3, 0) #master subnet is a smaller subnet within the vnet. i.e from /21 to /24
-  node_subnet_cidr   = cidrsubnet(var.machine_cidr, 3, 1) #node subnet is a smaller subnet within the vnet. i.e from /21 to /24
 }
 
 provider "azurerm" {
@@ -17,6 +14,13 @@ provider "azurerm" {
   tenant_id       = var.azure_tenant_id
 }
 
+provider "azureprivatedns" {
+  subscription_id = var.azure_subscription_id
+  client_id       = var.azure_client_id
+  client_secret   = var.azure_client_secret
+  tenant_id       = var.azure_tenant_id
+}
+
 module "bootstrap" {
   source              = "./bootstrap"
   resource_group_name = azurerm_resource_group.main.name
@@ -32,24 +36,15 @@ module "bootstrap" {
   tags                = local.tags
   storage_account     = azurerm_storage_account.cluster
   nsg_name            = module.vnet.master_nsg_name
-
-  # This is to create explicit dependency on private zone to exist before VMs are created in the vnet. https://github.com/MicrosoftDocs/azure-docs/issues/13728
-  private_dns_zone_id = azurerm_dns_zone.private.id
 }
 
 module "vnet" {
   source              = "./vnet"
-  vnet_name           = azurerm_virtual_network.cluster_vnet.name
   resource_group_name = azurerm_resource_group.main.name
   vnet_cidr           = var.machine_cidr
-  master_subnet_cidr  = local.master_subnet_cidr
-  node_subnet_cidr    = local.node_subnet_cidr
   cluster_id          = var.cluster_id
   region              = var.azure_region
   dns_label           = var.cluster_id
-
-  # This is to create explicit dependency on private zone to exist before VMs are created in the vnet. https://github.com/MicrosoftDocs/azure-docs/issues/13728
-  private_dns_zone_id = azurerm_dns_zone.private.id
 }
 
 module "master" {
@@ -66,24 +61,21 @@ module "master" {
   elb_backend_pool_id = module.vnet.public_lb_backend_pool_id
   ilb_backend_pool_id = module.vnet.internal_lb_backend_pool_id
   subnet_id           = module.vnet.public_subnet_id
-  master_subnet_cidr  = local.master_subnet_cidr
   instance_count      = var.master_count
   storage_account     = azurerm_storage_account.cluster
   os_volume_size      = var.azure_master_root_volume_size
-
-  # This is to create explicit dependency on private zone to exist before VMs are created in the vnet. https://github.com/MicrosoftDocs/azure-docs/issues/13728
-  private_dns_zone_id = azurerm_dns_zone.private.id
 }
 
 module "dns" {
   source                          = "./dns"
   cluster_domain                  = var.cluster_domain
+  cluster_id                      = var.cluster_id
   base_domain                     = var.base_domain
+  virtual_network                 = module.vnet.network_id
   external_lb_fqdn                = module.vnet.public_lb_pip_fqdn
   internal_lb_ipaddress           = module.vnet.internal_lb_ip_address
   resource_group_name             = azurerm_resource_group.main.name
   base_domain_resource_group_name = var.azure_base_domain_resource_group_name
-  private_dns_zone_name           = azurerm_dns_zone.private.name
   etcd_count                      = var.master_count
   etcd_ip_addresses               = module.master.ip_addresses
 }
@@ -121,21 +113,6 @@ resource "azurerm_role_assignment" "main" {
   principal_id         = azurerm_user_assigned_identity.main.principal_id
 }
 
-# https://github.com/MicrosoftDocs/azure-docs/issues/13728
-resource "azurerm_dns_zone" "private" {
-  name                           = var.cluster_domain
-  resource_group_name            = azurerm_resource_group.main.name
-  zone_type                      = "Private"
-  resolution_virtual_network_ids = [azurerm_virtual_network.cluster_vnet.id]
-}
-
-resource "azurerm_virtual_network" "cluster_vnet" {
-  name                = "${var.cluster_id}-vnet"
-  resource_group_name = azurerm_resource_group.main.name
-  location            = var.azure_region
-  address_space       = [var.machine_cidr]
-}
-
 # copy over the vhd to cluster resource group and create an image using that
 resource "azurerm_storage_container" "vhd" {
   name                 = "vhd"

data/data/azure/master/variables.tf

@@ -78,16 +78,6 @@ variable "ignition" {
   type = string
 }
 
-variable "master_subnet_cidr" {
-  type        = string
-  description = "the master subnet cidr"
-}
-
-variable "private_dns_zone_id" {
-  type        = string
-  description = "This is to create explicit dependency on private zone to exist before VMs are created in the vnet. https://github.com/MicrosoftDocs/azure-docs/issues/13728"
-}
-
 variable "availability_zones" {
   type        = list(string)
   description = "List of the availability zones in which to create the masters. The length of this list must match instance_count."

data/data/azure/vnet/common.tf

@@ -3,15 +3,11 @@
 
 // Only reference data sources which are guaranteed to exist at any time (above) in this locals{} block
 locals {
-  subnet_ids = azurerm_subnet.master_subnet.id
-
-  lb_fqdn = azurerm_lb.public.id
-
-  elb_backend_pool_id = azurerm_lb_backend_address_pool.master_public_lb_pool.id
-
-  internal_lb_controlplane_pool_id = azurerm_lb_backend_address_pool.internal_lb_controlplane_pool.id
-
-  public_lb_id   = azurerm_lb.public.id
-  internal_lb_id = azurerm_lb.internal.id
+  master_subnet_cidr = cidrsubnet(var.vnet_cidr, 3, 0) #master subnet is a smaller subnet within the vnet. i.e from /21 to /24
+  node_subnet_cidr   = cidrsubnet(var.vnet_cidr, 3, 1) #node subnet is a smaller subnet within the vnet. i.e from /21 to /24
 }
 
+data "azurerm_virtual_network" "cluster_vnet" {
+  name                = azurerm_virtual_network.cluster_vnet.name
+  resource_group_name = var.resource_group_name
+}

data/data/azure/vnet/internal-lb.tf

@@ -12,7 +12,7 @@ resource "azurerm_lb" "internal" {
     name                          = local.internal_lb_frontend_ip_configuration_name
     subnet_id                     = azurerm_subnet.master_subnet.id
     private_ip_address_allocation = "Static"
-    private_ip_address            = cidrhost(var.master_subnet_cidr, -2) #last ip is reserved by azure
+    private_ip_address            = cidrhost(local.master_subnet_cidr, -2) #last ip is reserved by azure
   }
 }
 

data/data/azure/vnet/outputs.tf

@@ -2,20 +2,24 @@ output "cluster-pip" {
   value = azurerm_public_ip.cluster_public_ip.ip_address
 }
 
+output "network_id" {
+  value = data.azurerm_virtual_network.cluster_vnet.id
+}
+
 output "public_subnet_id" {
-  value = local.subnet_ids
+  value = azurerm_subnet.master_subnet.id
 }
 
 output "public_lb_backend_pool_id" {
   value = azurerm_lb_backend_address_pool.master_public_lb_pool.id
 }
 
 output "internal_lb_backend_pool_id" {
-  value = local.internal_lb_controlplane_pool_id
+  value = azurerm_lb_backend_address_pool.internal_lb_controlplane_pool.id
 }
 
 output "public_lb_id" {
-  value = local.public_lb_id
+  value = azurerm_lb.public.id
 }
 
 output "public_lb_pip_fqdn" {

data/data/azure/vnet/variables.tf

@@ -1,21 +1,7 @@
-variable "vnet_name" {
-  type = string
-}
-
 variable "vnet_cidr" {
   type = string
 }
 
-variable "master_subnet_cidr" {
-  type        = string
-  description = "The subnet for the masters"
-}
-
-variable "node_subnet_cidr" {
-  type        = string
-  description = "The subnet for the workers"
-}
-
 variable "resource_group_name" {
   type        = string
   description = "Resource group for the deployment"
@@ -50,8 +36,3 @@ variable "dns_label" {
   type        = string
   description = "The label used to build the dns name. i.e. <label>.<region>.cloudapp.azure.com"
 }
-
-variable "private_dns_zone_id" {
-  type        = string
-  description = "This is to create explicit dependency on private zone to exist before VMs are created in the vnet. https://github.com/MicrosoftDocs/azure-docs/issues/13728"
-}

data/data/azure/vnet/vnet.tf

@@ -1,3 +1,10 @@
+resource "azurerm_virtual_network" "cluster_vnet" {
+  name                = "${var.cluster_id}-vnet"
+  resource_group_name = var.resource_group_name
+  location            = var.region
+  address_space       = [var.vnet_cidr]
+}
+
 resource "azurerm_route_table" "route_table" {
   name                = "${var.cluster_id}-node-routetable"
   location            = var.region
@@ -6,15 +13,15 @@ resource "azurerm_route_table" "route_table" {
 
 resource "azurerm_subnet" "master_subnet" {
   resource_group_name  = var.resource_group_name
-  address_prefix       = var.master_subnet_cidr
-  virtual_network_name = var.vnet_name
+  address_prefix       = local.master_subnet_cidr
+  virtual_network_name = data.azurerm_virtual_network.cluster_vnet.name
   name                 = "${var.cluster_id}-master-subnet"
 }
 
 resource "azurerm_subnet" "node_subnet" {
   resource_group_name  = var.resource_group_name
-  address_prefix       = var.node_subnet_cidr
-  virtual_network_name = var.vnet_name
+  address_prefix       = local.node_subnet_cidr
+  virtual_network_name = data.azurerm_virtual_network.cluster_vnet.name
   name                 = "${var.cluster_id}-worker-subnet"
 }
 

pkg/asset/installconfig/azure/dns.go

@@ -62,6 +62,16 @@ func (config DNSConfig) GetDNSZoneID(rgName string, zoneName string) string {
 		zoneName)
 }
 
+//GetPrivateDNSZoneID returns the Azure Private DNS zone resourceID
+//by interpolating the subscriptionID, the resource group and the zone name
+func (config DNSConfig) GetPrivateDNSZoneID(rgName string, zoneName string) string {
+	return fmt.Sprintf(
+		"/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/privateDnsZones/%s",
+		config.Session.Credentials.SubscriptionID,
+		rgName,
+		zoneName)
+}
+
 //GetDNSZone returns a DNS zone selected by survey
 func (config DNSConfig) GetDNSZone() (*Zone, error) {
 	//call azure api using the session to retrieve available base domain

pkg/asset/manifests/dns.go

@@ -98,7 +98,7 @@ func (d *DNS) Generate(dependencies asset.Parents) error {
 			ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain),
 		}
 		config.Spec.PrivateZone = &configv1.DNSZone{
-			ID: dnsConfig.GetDNSZoneID(clusterID.InfraID+"-rg", installConfig.Config.ClusterDomain()),
+			ID: dnsConfig.GetPrivateDNSZoneID(clusterID.InfraID+"-rg", installConfig.Config.ClusterDomain()),
 		}
 	case gcptypes.Name:
 		zone, err := icgcp.GetPublicZone(context.TODO(), installConfig.Config.Platform.GCP.ProjectID, installConfig.Config.BaseDomain)