Power VS: Enable IBM DNS service for private cluster support

data/data/powervs/cluster/dns/dns.tf

@@ -1,28 +1,68 @@
+############
+# Public DNS
+############
+
 data "ibm_cis_domain" "base_domain" {
-  cis_id = var.cis_id
+  count  = var.publish_strategy == "Internal" ? 0 : 1
+  cis_id = var.service_id
   domain = var.base_domain
 }
 
 resource "ibm_cis_dns_record" "kubernetes_api" {
-  cis_id    = var.cis_id
-  domain_id = data.ibm_cis_domain.base_domain.id
+  count     = var.publish_strategy == "Internal" ? 0 : 1
+  cis_id    = var.service_id
+  domain_id = data.ibm_cis_domain.base_domain[count.index].id
   type      = "CNAME"
   name      = "api.${var.cluster_domain}"
   content   = var.load_balancer_hostname
   ttl       = 60
 }
 
 resource "ibm_cis_dns_record" "kubernetes_api_internal" {
-  cis_id    = var.cis_id
-  domain_id = data.ibm_cis_domain.base_domain.id
+  count     = var.publish_strategy == "Internal" ? 0 : 1
+  cis_id    = var.service_id
+  domain_id = data.ibm_cis_domain.base_domain[count.index].id
   type      = "CNAME"
   name      = "api-int.${var.cluster_domain}"
   content   = var.load_balancer_int_hostname
   ttl       = 60
 }
 
-locals {
-  proxy_count = var.publish_strategy == "Internal" ? 1 : 0
+#############
+# Private DNS
+#############
+
+data "ibm_dns_zones" "dns_zones" {
+  count       = var.publish_strategy == "Internal" ? 1 : 0
+  instance_id = var.service_id
+}
+
+resource "ibm_dns_permitted_network" "permit_vpc_network_for_dns" {
+  count       = var.publish_strategy == "Internal" ? 1 : 0
+  instance_id = var.service_id
+  zone_id     = local.dns_zone.zone_id
+  vpc_crn     = var.vpc_crn
+  type        = "vpc"
+}
+
+resource "ibm_dns_resource_record" "kubernetes_api" {
+  count       = var.publish_strategy == "Internal" ? 1 : 0
+  instance_id = var.service_id
+  zone_id     = local.dns_zone.zone_id
+  type        = "CNAME"
+  name        = "api.${var.cluster_domain}"
+  rdata       = var.load_balancer_int_hostname
+  ttl         = 60
+}
+
+resource "ibm_dns_resource_record" "kubernetes_api_internal" {
+  count       = var.publish_strategy == "Internal" ? 1 : 0
+  instance_id = var.service_id
+  zone_id     = local.dns_zone.zone_id
+  type        = "CNAME"
+  name        = "api-int.${var.cluster_domain}"
+  rdata       = var.load_balancer_int_hostname
+  ttl         = 60
 }
 
 resource "ibm_is_ssh_key" "dns_ssh_key" {
@@ -76,7 +116,10 @@ data "ibm_is_image" "dns_vm_image" {
   name  = var.dns_vm_image_name
 }
 
+
 locals {
+  dns_zone         = var.publish_strategy == "Internal" ? data.ibm_dns_zones.dns_zones[0].dns_zones[index(data.ibm_dns_zones.dns_zones[0].dns_zones.*.name, var.base_domain)] : null
+  proxy_count      = var.publish_strategy == "Internal" ? 1 : 0
   user_data_string = <<EOF
 #cloud-config
 packages:
@@ -100,10 +143,6 @@ runcmd:
 EOF
 }
 
-#
-# The following is because ci/prow/tf-fmt is recommending that
-# style of formatting which seems like a bug to me.
-#
 resource "ibm_is_instance" "dns_vm_vsi" {
   count = local.proxy_count
   name = "${var.cluster_id}-dns-vsi"

data/data/powervs/cluster/dns/outputs.tf

@@ -0,0 +1,3 @@
+output "dns_server" {
+  value = var.publish_strategy == "Internal" ? ibm_is_instance.dns_vm_vsi[0].primary_network_interface[0].primary_ipv4_address : "1.1.1.1"
+}

data/data/powervs/cluster/dns/variables.tf

@@ -1,6 +1,6 @@
-variable "cis_id" {
+variable "service_id" {
   type        = string
-  description = "The ID of the IBM Cloud CIS instance that will be used for the DNS records."
+  description = "The ID of the IBM Cloud CIS instance, or IBM Cloud DNS instance, that will be used for the DNS records."
 }
 
 variable "base_domain" {
@@ -28,6 +28,11 @@ variable "cluster_id" {
   description = "The ID created by the installer to uniquely identify the created cluster."
 }
 
+variable "vpc_crn" {
+  type        = string
+  description = "The CRN of the VPC."
+}
+
 variable "vpc_id" {
   type        = string
   description = "The ID of the VPC."
@@ -60,3 +65,4 @@ variable "publish_strategy" {
   description = "The cluster publishing strategy, either Internal or External"
   default     = "External"
 }
+

data/data/powervs/cluster/main.tf

@@ -39,6 +39,7 @@ module "pi_network" {
   machine_cidr      = var.machine_v4_cidrs[0]
   cloud_conn_name   = var.powervs_ccon_name
   vpc_crn           = module.vpc.vpc_crn
+  dns_server        = module.dns.dns_server
 }
 
 resource "ibm_pi_key" "cluster_key" {
@@ -62,7 +63,7 @@ module "master" {
   powervs_region      = var.powervs_region
   powervs_zone        = var.powervs_zone
   vpc_region          = var.powervs_vpc_region
-  vpc_zone            = var.powervs_vpc_zone
+  vpc_zone            = module.vpc.vpc_zone
   memory              = var.powervs_master_memory
   processors          = var.powervs_master_processors
   ignition            = var.ignition_master
@@ -110,21 +111,26 @@ module "loadbalancer" {
   vpc_subnet_id  = module.vpc.vpc_subnet_id
 }
 
+locals {
+  dns_service_id = var.powervs_publish_strategy == "Internal" ? var.powervs_dns_guid : var.powervs_cis_crn
+}
+
 module "dns" {
   providers = {
     ibm = ibm.vpc
   }
   source = "./dns"
 
-  cis_id                     = var.powervs_cis_crn
+  service_id                 = local.dns_service_id
   base_domain                = var.base_domain
   cluster_domain             = var.cluster_domain
   load_balancer_hostname     = module.loadbalancer.lb_hostname
   load_balancer_int_hostname = module.loadbalancer.lb_int_hostname
   cluster_id                 = var.cluster_id
+  vpc_crn                    = module.vpc.vpc_crn
   vpc_id                     = module.vpc.vpc_id
   vpc_subnet_id              = module.vpc.vpc_subnet_id
-  vpc_zone                   = var.powervs_vpc_zone
+  vpc_zone                   = module.vpc.vpc_zone
   ssh_key                    = var.powervs_ssh_key
   publish_strategy           = var.powervs_publish_strategy
   # dns_vm_image_name        = @FUTURE

data/data/powervs/cluster/power_network/outputs.tf

@@ -4,4 +4,4 @@ output "dhcp_id" {
 
 output "dhcp_network_id" {
   value = data.ibm_pi_dhcp.dhcp_service.network_id
-}
+}

data/data/powervs/cluster/power_network/pi_network.tf

@@ -12,8 +12,8 @@ resource "ibm_pi_dhcp" "new_dhcp_service" {
   count                  = var.pvs_network_name == "" ? 1 : 0
   pi_cloud_instance_id   = var.cloud_instance_id
   pi_cloud_connection_id = data.ibm_pi_cloud_connection.cloud_connection.id
-  pi_dns_server          = "1.1.1.1"
   pi_cidr                = var.machine_cidr
+  pi_dns_server          = var.dns_server
   # the pi_dhcp_name param will be prefixed by the DHCP ID when created, so keep it short here:
   pi_dhcp_name = var.cluster_id
 }

data/data/powervs/cluster/power_network/variables.tf

@@ -34,3 +34,9 @@ variable "machine_cidr" {
   type        = string
   description = "The machine network (IPv4 only)"
 }
+
+variable "dns_server" {
+  type        = string
+  description = "The desired DNS server for the DHCP instance to server."
+  default     = "1.1.1.1"
+}

data/data/powervs/cluster/vpc/outputs.tf

@@ -6,6 +6,10 @@ output "vpc_subnet_id" {
   value = data.ibm_is_subnet.ocp_vpc_subnet.id
 }
 
+output "vpc_zone" {
+  value = data.ibm_is_subnet.ocp_vpc_subnet.zone
+}
+
 output "vpc_crn" {
   value = data.ibm_is_vpc.ocp_vpc.crn
 }

data/data/powervs/cluster/vpc/vpc.tf

@@ -17,22 +17,26 @@ resource "time_sleep" "wait_for_vpc" {
 
 resource "ibm_is_public_gateway" "dns_vm_gateway" {
   name = "${var.cluster_id}-gateway"
-  vpc  = ibm_is_vpc.new_vpc[0].id
-  zone = var.vpc_zone
+  vpc  = data.ibm_is_vpc.ocp_vpc.id
+  zone = data.ibm_is_subnet.ocp_vpc_subnet.zone
 }
 
 resource "ibm_is_subnet" "new_vpc_subnet" {
   count                    = var.vpc_subnet_name == "" ? 1 : 0
   depends_on               = [time_sleep.wait_for_vpc]
   name                     = "vpc-subnet-${var.cluster_id}"
-  vpc                      = ibm_is_vpc.new_vpc[0].id
+  vpc                      = data.ibm_is_vpc.ocp_vpc.id
   resource_group           = data.ibm_resource_group.rg.id
   total_ipv4_address_count = 256
   zone                     = var.vpc_zone
-  public_gateway           = ibm_is_public_gateway.dns_vm_gateway.id
   tags                     = [var.cluster_id]
 }
 
+resource "ibm_is_subnet_public_gateway_attachment" "subnet_public_gateway_attachment" {
+  subnet         = data.ibm_is_subnet.ocp_vpc_subnet.id
+  public_gateway = ibm_is_public_gateway.dns_vm_gateway.id
+}
+
 data "ibm_is_vpc" "ocp_vpc" {
   name = var.vpc_name == "" ? ibm_is_vpc.new_vpc[0].name : var.vpc_name
 }

data/data/powervs/variables-powervs.tf

@@ -43,10 +43,6 @@ variable "powervs_publish_strategy" {
   type        = string
   description = "The cluster publishing strategy, either Internal or External"
   default     = "External"
-  validation {
-    condition     = var.powervs_publish_strategy == "External" || var.powervs_publish_strategy == "Internal"
-    error_message = "The powervs_publish_strategy value must be \"External\" or \"Internal\"."
-  }
 }
 
 ################################################################
@@ -186,6 +182,11 @@ variable "powervs_cis_crn" {
   description = "The CRN of CIS instance to use."
 }
 
+variable "powervs_dns_guid" {
+  type        = string
+  description = "The GUID of the IBM DNS Service instance to use when creating a private cluster."
+}
+
 ################################################################
 # Output Variables
 ################################################################

pkg/asset/cluster/powervs/powervs.go

@@ -12,11 +12,13 @@ import (
 // Metadata converts an install configuration to PowerVS metadata.
 func Metadata(config *types.InstallConfig, meta *icpowervs.Metadata) *powervs.Metadata {
 	cisCRN, _ := meta.CISInstanceCRN(context.TODO())
+	dnsCRN, _ := meta.DNSInstanceCRN(context.TODO())
 
 	return &powervs.Metadata{
 		BaseDomain:           config.BaseDomain,
 		PowerVSResourceGroup: config.Platform.PowerVS.PowerVSResourceGroup,
 		CISInstanceCRN:       cisCRN,
+		DNSInstanceCRN:       dnsCRN,
 		Region:               config.Platform.PowerVS.Region,
 		VPCRegion:            config.Platform.PowerVS.VPCRegion,
 		Zone:                 config.Platform.PowerVS.Zone,

pkg/asset/cluster/tfvars.go

@@ -748,10 +748,22 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			return err
 		}
 
-		// Get CISInstanceCRN from InstallConfig metadata
-		crn, err := installConfig.PowerVS.CISInstanceCRN(ctx)
-		if err != nil {
-			return err
+		var cisCRN, dnsCRN string
+		switch installConfig.Config.Publish {
+		case types.InternalPublishingStrategy:
+			// Get DNSInstanceCRN from InstallConfig metadata
+			dnsCRN, err = installConfig.PowerVS.DNSInstanceCRN(ctx)
+			if err != nil {
+				return err
+			}
+		case types.ExternalPublishingStrategy:
+			// Get CISInstanceCRN from InstallConfig metadata
+			cisCRN, err = installConfig.PowerVS.CISInstanceCRN(ctx)
+			if err != nil {
+				return err
+			}
+		default:
+			return errors.New("unknown publishing strategy")
 		}
 
 		masterConfigs := make([]*machinev1.PowerVSMachineProviderConfig, len(masters))
@@ -779,7 +791,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				VPCName:              installConfig.Config.PowerVS.VPCName,
 				VPCSubnetName:        vpcSubnet,
 				CloudConnectionName:  installConfig.Config.PowerVS.CloudConnectionName,
-				CISInstanceCRN:       crn,
+				CISInstanceCRN:       cisCRN,
+				DNSInstanceCRN:       dnsCRN,
 				PublishStrategy:      installConfig.Config.Publish,
 			},
 		)

pkg/asset/installconfig/platformprovisioncheck.go

@@ -133,7 +133,7 @@ func (a *PlatformProvisionCheck) Generate(dependencies asset.Parents) error {
 		if err != nil {
 			return err
 		}
-		err = powervsconfig.ValidatePreExistingPublicDNS(client, ic.Config, ic.PowerVS)
+		err = powervsconfig.ValidatePreExistingDNS(client, ic.Config, ic.PowerVS)
 	case libvirt.Name, none.Name:
 		// no special provisioning requirements to check
 	case nutanix.Name: