Add support for private DNS server to PowerVS

data/data/powervs/bootstrap/vm/outputs.tf

@@ -1,4 +1,4 @@
-output bootstrap_ip {
+output "bootstrap_ip" {
   value = local.bootstrap_ips[0]
 }
 

data/data/powervs/cluster/bootstrap/lb/ouputs.tf

@@ -1,7 +1,7 @@
-output api_member_ext_id {
+output "api_member_ext_id" {
   value = ibm_is_lb_pool_member.api_member.id
 }
 
-output api_member_int_id {
+output "api_member_int_id" {
   value = ibm_is_lb_pool_member.api_member_int.id
 }

data/data/powervs/cluster/bootstrap/outputs.tf

@@ -1,11 +1,11 @@
-output bootstrap_ip {
+output "bootstrap_ip" {
   value = module.vm.bootstrap_ip
 }
 
-output api_member_ext_id {
+output "api_member_ext_id" {
   value = module.lb.api_member_ext_id
 }
 
-output api_member_int_id {
+output "api_member_int_id" {
   value = module.lb.api_member_int_id
 }

data/data/powervs/cluster/dns/dns.tf

@@ -20,3 +20,78 @@ resource "ibm_cis_dns_record" "kubernetes_api_internal" {
   content   = var.load_balancer_int_hostname
   ttl       = 60
 }
+
+locals {
+  proxy_count = var.publish_strategy == "Internal" ? 1 : 0
+}
+
+resource "ibm_is_ssh_key" "dns_ssh_key" {
+  count      = local.proxy_count
+  name       = "${var.cluster_id}-dns-ssh-key"
+  public_key = var.ssh_key
+}
+
+resource "ibm_is_security_group" "dns_vm_sg" {
+  count = local.proxy_count
+  name  = "${var.cluster_id}-dns-sg"
+  vpc   = var.vpc_id
+}
+
+# allow all outgoing network traffic
+resource "ibm_is_security_group_rule" "dns_vm_sg_outgoing_all" {
+  count     = local.proxy_count
+  group     = ibm_is_security_group.dns_vm_sg[0].id
+  direction = "outbound"
+  remote    = "0.0.0.0/0"
+}
+
+# allow all incoming network traffic on port 22
+resource "ibm_is_security_group_rule" "dns_vm_sg_ssh_all" {
+  count     = local.proxy_count
+  group     = ibm_is_security_group.dns_vm_sg[0].id
+  direction = "inbound"
+  remote    = "0.0.0.0/0"
+
+  tcp {
+    port_min = 22
+    port_max = 22
+  }
+}
+
+data "ibm_is_image" "dns_vm_image" {
+  count = local.proxy_count
+  name  = var.dns_vm_image_name
+}
+
+locals {
+  user_data_string = <<EOF
+#cloud-config
+packages:
+  - bind
+  - bind-utils
+runcmd:
+  - systemctl enable named.service
+  - systemctl start named.service
+EOF
+}
+
+#
+# The following is because ci/prow/tf-fmt is recommending that
+# style of formatting which seems like a bug to me.
+#
+resource "ibm_is_instance" "dns_vm_vsi" {
+  count = local.proxy_count
+  name = "${var.cluster_id}-dns-vsi"
+  vpc = var.vpc_id
+  zone = var.vpc_zone
+  keys = [ibm_is_ssh_key.dns_ssh_key[0].id]
+  image = data.ibm_is_image.dns_vm_image[0].id
+  profile = "cx2-2x4"
+
+  primary_network_interface {
+    subnet = var.vpc_subnet_id
+    security_groups = [ibm_is_security_group.dns_vm_sg[0].id]
+  }
+
+  user_data = base64encode(local.user_data_string)
+}

data/data/powervs/cluster/dns/variables.tf

@@ -23,6 +23,40 @@ variable "load_balancer_int_hostname" {
   description = "The hostname for the internal load balancer."
 }
 
+variable "cluster_id" {
+  type        = string
+  description = "The ID created by the installer to uniquely identify the created cluster."
+}
 
+variable "vpc_id" {
+  type        = string
+  description = "The ID of the VPC."
+}
 
+variable "vpc_subnet_id" {
+  type        = string
+  description = "The ID of the VPC subnet."
+}
+
+variable "vpc_zone" {
+  type        = string
+  description = "The IBM Cloud zone in which the VPC is created."
+}
 
+variable "dns_vm_image_name" {
+  type        = string
+  description = "The image name for the DNS VM."
+  default     = "ibm-centos-7-9-minimal-amd64-6"
+}
+
+variable "ssh_key" {
+  type        = string
+  description = "Public key for keypair used to access cluster. Required when creating 'ibm_pi_instance' resources."
+  default     = ""
+}
+
+variable "publish_strategy" {
+  type        = string
+  description = "The cluster publishing strategy, either Internal or External"
+  default     = "External"
+}

data/data/powervs/cluster/main.tf

@@ -120,4 +120,11 @@ module "dns" {
   cluster_domain             = var.cluster_domain
   load_balancer_hostname     = module.loadbalancer.lb_hostname
   load_balancer_int_hostname = module.loadbalancer.lb_int_hostname
+  cluster_id                 = var.cluster_id
+  vpc_id                     = module.vpc.vpc_id
+  vpc_subnet_id              = module.vpc.vpc_subnet_id
+  vpc_zone                   = var.powervs_vpc_zone
+  ssh_key                    = var.powervs_ssh_key
+  publish_strategy           = var.powervs_publish_strategy
+  # dns_vm_image_name        = @FUTURE
 }

data/data/powervs/cluster/master/lb/variables.tf

@@ -1,5 +1,5 @@
 variable "master_ips" {
-  type        = list
+  type        = list(any)
   description = "The IP addresses of the master nodes."
 }
 

data/data/powervs/variables-powervs.tf

@@ -39,6 +39,16 @@ variable "powervs_cloud_instance_id" {
   description = "The cloud instance ID of your account"
 }
 
+variable "powervs_publish_strategy" {
+  type        = string
+  description = "The cluster publishing strategy, either Internal or External"
+  default     = "External"
+  validation {
+    condition     = var.powervs_publish_strategy == "External" || var.powervs_publish_strategy == "Internal"
+    error_message = "The powervs_publish_strategy value must be \"External\" or \"Internal\"."
+  }
+}
+
 ################################################################
 # Configure storage
 ################################################################

pkg/asset/cluster/tfvars.go

@@ -732,6 +732,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				VPCSubnetName:        vpcSubnet,
 				CloudConnectionName:  installConfig.Config.PowerVS.CloudConnectionName,
 				CISInstanceCRN:       crn,
+				PublishStrategy:      installConfig.Config.Publish,
 			},
 		)
 		if err != nil {

pkg/destroy/powervs/cloud-instance.go

@@ -0,0 +1,137 @@
+package powervs
+
+import (
+	"context"
+	"github.com/IBM/go-sdk-core/v5/core"
+	"github.com/IBM/vpc-go-sdk/vpcv1"
+	"github.com/pkg/errors"
+	"strings"
+)
+
+const (
+	cloudInstanceTypeName = "cloudInstance"
+)
+
+// listCloudInstances lists instances in the cloud server.
+func (o *ClusterUninstaller) listCloudInstances() (cloudResources, error) {
+	o.Logger.Debugf("Listing virtual Cloud service instances")
+
+	ctx, _ := o.contextWithTimeout()
+
+	options := o.vpcSvc.NewListInstancesOptions()
+
+	// https://raw.githubusercontent.com/IBM/vpc-go-sdk/master/vpcv1/vpc_v1.go
+	resources, _, err := o.vpcSvc.ListInstancesWithContext(ctx, options)
+	if err != nil {
+		o.Logger.Warnf("Error o.vpcSvc.ListInstancesWithContext: %v", err)
+		return nil, err
+	}
+
+	var foundOne = false
+
+	result := []cloudResource{}
+	for _, instance := range resources.Instances {
+		if strings.Contains(*instance.Name, o.InfraID) {
+			foundOne = true
+			o.Logger.Debugf("listCloudInstances: FOUND: %s, %s, %s", *instance.ID, *instance.Name, *instance.Status)
+			result = append(result, cloudResource{
+				key:      *instance.ID,
+				name:     *instance.Name,
+				status:   *instance.Status,
+				typeName: cloudInstanceTypeName,
+				id:       *instance.ID,
+			})
+		}
+	}
+	if !foundOne {
+		o.Logger.Debugf("listCloudInstances: NO matching virtual instance against: %s", o.InfraID)
+		for _, instance := range resources.Instances {
+			o.Logger.Debugf("listInstances: only found virtual instance: %s", *instance.Name)
+		}
+	}
+
+	return cloudResources{}.insert(result...), nil
+}
+
+// destroyCloudInstance deletes a given instance.
+func (o *ClusterUninstaller) destroyCloudInstance(item cloudResource) error {
+	var (
+		ctx                   context.Context
+		err                   error
+		getInstanceOptions    *vpcv1.GetInstanceOptions
+		deleteInstanceOptions *vpcv1.DeleteInstanceOptions
+		response              *core.DetailedResponse
+	)
+
+	ctx, _ = o.contextWithTimeout()
+
+	getInstanceOptions = o.vpcSvc.NewGetInstanceOptions(item.id)
+
+	_, _, err = o.vpcSvc.GetInstanceWithContext(ctx, getInstanceOptions)
+	if err != nil {
+		o.deletePendingItems(item.typeName, []cloudResource{item})
+		o.Logger.Infof("Deleted Cloud instance %q", item.name)
+		return nil
+	}
+
+	o.Logger.Debugf("Deleting Cloud instance %q", item.name)
+
+	deleteInstanceOptions = o.vpcSvc.NewDeleteInstanceOptions(item.id)
+
+	response, err = o.vpcSvc.DeleteInstanceWithContext(ctx, deleteInstanceOptions)
+	if err != nil {
+		o.Logger.Infof("Error: o.vpcSvc.DeleteInstanceWithContext: %q %q", err, response)
+		return err
+	}
+
+	o.deletePendingItems(item.typeName, []cloudResource{item})
+	o.Logger.Infof("Deleted Cloud instance %q", item.name)
+
+	return nil
+}
+
+// destroyCloudInstances searches for Cloud instances that have a name that starts with
+// the cluster's infra ID.
+func (o *ClusterUninstaller) destroyCloudInstances() error {
+	found, err := o.listCloudInstances()
+	if err != nil {
+		return err
+	}
+
+	items := o.insertPendingItems(cloudInstanceTypeName, found.list())
+
+	ctx, _ := o.contextWithTimeout()
+
+	for !o.timeout(ctx) {
+		for _, item := range items {
+			select {
+			case <-o.Context.Done():
+				o.Logger.Debugf("destroyCloudInstances: case <-o.Context.Done()")
+				return o.Context.Err() // we're cancelled, abort
+			default:
+			}
+
+			if _, ok := found[item.key]; !ok {
+				// This item has finished deletion.
+				o.deletePendingItems(item.typeName, []cloudResource{item})
+				o.Logger.Infof("Deleted Cloud instance %q", item.name)
+				continue
+			}
+			err := o.destroyCloudInstance(item)
+			if err != nil {
+				o.errorTracker.suppressWarning(item.key, err, o.Logger)
+			}
+		}
+
+		items = o.getPendingItems(cloudInstanceTypeName)
+		if len(items) == 0 {
+			break
+		}
+	}
+
+	if items = o.getPendingItems(cloudInstanceTypeName); len(items) > 0 {
+		return errors.Errorf("destroyCloudInstances: %d undeleted items pending", len(items))
+	}
+
+	return nil
+}