Power VS: Disconnected cluster support

data/data/powervs/bootstrap/main.tf

@@ -21,7 +21,7 @@ module "vm" {
   resource_group        = var.powervs_resource_group
   cluster_id            = var.cluster_id
   ssh_key_id            = var.cluster_key_id
-  cos_bucket_location   = var.powervs_cos_bucket_location
+  cos_bucket_location   = var.powervs_vpc_region
   cos_instance_location = var.powervs_cos_instance_location
   cos_storage_class     = var.powervs_cos_storage_class
   ignition              = var.ignition_bootstrap
@@ -33,6 +33,8 @@ module "vm" {
   cloud_instance_id     = var.powervs_cloud_instance_id
   dhcp_network_id       = var.dhcp_network_id
   dhcp_id               = var.dhcp_id
+  proxy_server_ip       = var.proxy_server_ip
+  enable_snat           = var.powervs_enable_snat
 }
 
 module "lb" {

data/data/powervs/bootstrap/variables.tf

@@ -8,3 +8,4 @@ variable "machine_cfg_pool_id" { type = string }
 variable "api_pool_int_id" { type = string }
 variable "api_pool_ext_id" { type = string }
 variable "boot_image_id" { type = string }
+variable "proxy_server_ip" { type = string }

data/data/powervs/bootstrap/vm/main.tf

@@ -50,7 +50,8 @@ resource "ibm_pi_instance" "bootstrap" {
     network_id = var.dhcp_network_id
   }
   pi_user_data = base64encode(templatefile("${path.module}/templates/bootstrap.ign", {
-    HOSTNAME    = ibm_cos_bucket.ignition.s3_endpoint_public
+    PROTOCOL    = var.enable_snat ? "https" : "http"
+    HOSTNAME    = var.enable_snat ? ibm_cos_bucket.ignition.s3_endpoint_public : var.proxy_server_ip
     BUCKET_NAME = ibm_cos_bucket.ignition.bucket_name
     OBJECT_NAME = ibm_cos_bucket_object.ignition.key
     IAM_TOKEN   = data.ibm_iam_auth_token.iam_token.iam_access_token

data/data/powervs/bootstrap/vm/templates/bootstrap.ign

@@ -3,7 +3,7 @@
     "version": "3.2.0",
     "config": {
       "replace": {
-        "source": "https://${HOSTNAME}/${BUCKET_NAME}/${OBJECT_NAME}",
+        "source": "${PROTOCOL}://${HOSTNAME}/${BUCKET_NAME}/${OBJECT_NAME}",
         "httpHeaders": [
           {
             "name": "Authorization",
@@ -13,4 +13,4 @@
       }
     }
   }
-}
+}

data/data/powervs/bootstrap/vm/variables.tf

@@ -71,3 +71,14 @@ variable "dhcp_network_id" {
   type        = string
   description = "The ID of the Power VS DHCP network."
 }
+
+variable "proxy_server_ip" {
+  type        = string
+  description = "The IP of the VPC VM serving as a service proxy."
+}
+
+variable "enable_snat" {
+  type        = bool
+  description = "Indicates if the DHCP server will have SNAT enabled."
+  default     = true
+}

data/data/powervs/cluster/dns/dns.tf

@@ -65,6 +65,16 @@ resource "ibm_dns_resource_record" "kubernetes_api_internal" {
   ttl         = 60
 }
 
+resource "ibm_dns_resource_record" "proxy_vsi_record" {
+  count       = var.publish_strategy == "Internal" ? 1 : 0
+  instance_id = var.service_id
+  zone_id     = local.dns_zone.zone_id
+  type        = "A"
+  name        = "proxy.${var.cluster_domain}"
+  rdata       = ibm_is_instance.dns_vm_vsi[0].primary_network_interface[0].primary_ipv4_address
+  ttl         = 60
+}
+
 resource "ibm_is_ssh_key" "dns_ssh_key" {
   count      = local.proxy_count
   name       = "${var.cluster_id}-dns-ssh-key"
@@ -111,51 +121,56 @@ resource "ibm_is_security_group_rule" "dns_vm_sg_dns_all" {
   }
 }
 
+# allow all incoming network traffic on port 80
+resource "ibm_is_security_group_rule" "dns_vm_sg_http_all" {
+  count     = local.proxy_count
+  group     = ibm_is_security_group.dns_vm_sg[0].id
+  direction = "inbound"
+  remote    = "0.0.0.0/0"
+
+  tcp {
+    port_min = 80
+    port_max = 80
+  }
+}
+
+# allow all incoming network traffic on port 3128 for squid proxy
+resource "ibm_is_security_group_rule" "dns_vm_sg_squid_all" {
+  count     = local.proxy_count
+  group     = ibm_is_security_group.dns_vm_sg[0].id
+  direction = "inbound"
+  remote    = "0.0.0.0/0"
+
+  tcp {
+    port_min = 3128
+    port_max = 3128
+  }
+}
+
 data "ibm_is_image" "dns_vm_image" {
   count = local.proxy_count
   name  = var.dns_vm_image_name
 }
 
 
 locals {
-  dns_zone         = var.publish_strategy == "Internal" ? data.ibm_dns_zones.dns_zones[0].dns_zones[index(data.ibm_dns_zones.dns_zones[0].dns_zones.*.name, var.base_domain)] : null
-  proxy_count      = var.publish_strategy == "Internal" ? 1 : 0
-  user_data_string = <<EOF
-#cloud-config
-packages:
-  - bind
-  - bind-utils
-write_files:
-- path: /tmp/named-conf-edit.sed
-  permissions: '0640'
-  content: |
-    /^\s*listen-on port 53 /s/127\.0\.0\.1/127\.0\.0\.1; MYIP/
-    /^\s*allow-query /s/localhost/any/
-    /^\s*dnssec-validation /s/ yes/ no/
-    /^\s*type hint;/s/ hint/ forward/
-    /^\s*file\s"named.ca";/d
-    /^\s*type forward/a \\tforward only;\n\tforwarders { 161.26.0.7; 161.26.0.8; };
-runcmd:
-  - export MYIP=`hostname -I`; sed -i.bak "s/MYIP/$MYIP/" /tmp/named-conf-edit.sed
-  - sed -i.orig -f /tmp/named-conf-edit.sed /etc/named.conf
-  - systemctl enable named.service
-  - systemctl start named.service
-EOF
+  dns_zone    = var.publish_strategy == "Internal" ? data.ibm_dns_zones.dns_zones[0].dns_zones[index(data.ibm_dns_zones.dns_zones[0].dns_zones.*.name, var.base_domain)] : null
+  proxy_count = var.publish_strategy == "Internal" ? 1 : 0
 }
 
 resource "ibm_is_instance" "dns_vm_vsi" {
-  count = local.proxy_count
-  name = "${var.cluster_id}-dns-vsi"
-  vpc = var.vpc_id
-  zone = var.vpc_zone
-  keys = [ibm_is_ssh_key.dns_ssh_key[0].id]
-  image = data.ibm_is_image.dns_vm_image[0].id
+  count   = local.proxy_count
+  name    = "${var.cluster_id}-dns-vsi"
+  vpc     = var.vpc_id
+  zone    = var.vpc_zone
+  keys    = [ibm_is_ssh_key.dns_ssh_key[0].id]
+  image   = data.ibm_is_image.dns_vm_image[0].id
   profile = "cx2-2x4"
 
   primary_network_interface {
-    subnet = var.vpc_subnet_id
+    subnet          = var.vpc_subnet_id
     security_groups = [ibm_is_security_group.dns_vm_sg[0].id]
   }
 
-  user_data = local.user_data_string
+  user_data = templatefile("${path.module}/templates/cloud-init.yaml.tpl", { is_proxy : ! var.enable_snat, vpc_region : var.vpc_region })
 }

data/data/powervs/cluster/dns/templates/cloud-init.yaml.tpl

@@ -0,0 +1,35 @@
+
+#cloud-config
+packages:
+  - bind
+  - bind-utils
+%{ if is_proxy ~}
+  - httpd
+  - mod_ssl
+  - squid
+%{ endif ~}
+write_files:
+%{ if is_proxy ~}
+- path: /etc/httpd/conf.d/proxy.conf
+  content: |
+    SSLProxyEngine on
+    ProxyPass / https://s3.direct.${vpc_region}.cloud-object-storage.appdomain.cloud/
+%{ endif ~}
+- path: /tmp/named-conf-edit.sed
+  permissions: '0640'
+  content: |
+    /^\s*listen-on port 53 /s/127\.0\.0\.1/127\.0\.0\.1; MYIP/
+    /^\s*allow-query /s/localhost/any/
+    /^\s*dnssec-validation /s/ yes/ no/
+    /^\s*type hint;/s/ hint/ forward/
+    /^\s*file\s"named.ca";/d
+    /^\s*type forward/a \\tforward only;\n\tforwarders { 161.26.0.7; 161.26.0.8; };
+runcmd:
+  - export MYIP=`hostname -I`; sed -i.bak "s/MYIP/$MYIP/" /tmp/named-conf-edit.sed
+  - sed -i.orig -f /tmp/named-conf-edit.sed /etc/named.conf
+  - systemctl enable named.service
+  - systemctl start named.service
+%{ if is_proxy ~}
+  - service httpd start
+  - service squid start
+%{ endif ~}

data/data/powervs/cluster/dns/variables.tf

@@ -43,6 +43,11 @@ variable "vpc_subnet_id" {
   description = "The ID of the VPC subnet."
 }
 
+variable "vpc_region" {
+  type        = string
+  description = "The IBM Cloud region in which the VPC is created."
+}
+
 variable "vpc_zone" {
   type        = string
   description = "The IBM Cloud zone in which the VPC is created."
@@ -66,3 +71,8 @@ variable "publish_strategy" {
   default     = "External"
 }
 
+variable "enable_snat" {
+  type        = bool
+  description = "Indicates if SNAT will be enabled for the DHCP service."
+  default     = true
+}

data/data/powervs/cluster/main.tf

@@ -131,7 +131,9 @@ module "dns" {
   vpc_id                     = module.vpc.vpc_id
   vpc_subnet_id              = module.vpc.vpc_subnet_id
   vpc_zone                   = module.vpc.vpc_zone
+  vpc_region                 = var.powervs_vpc_region
   ssh_key                    = var.powervs_ssh_key
   publish_strategy           = var.powervs_publish_strategy
+  enable_snat                = var.powervs_enable_snat
   # dns_vm_image_name        = @FUTURE
 }

data/data/powervs/cluster/outputs.tf

@@ -38,6 +38,10 @@ output "dhcp_network_id" {
   value = module.pi_network.dhcp_network_id
 }
 
+output "proxy_server_ip" {
+  value = module.dns.dns_server
+}
+
 output "vpc_id" {
   value = module.vpc.vpc_id
 }

data/data/powervs/cluster/power_network/pi_network.tf

@@ -14,6 +14,7 @@ resource "ibm_pi_dhcp" "new_dhcp_service" {
   pi_cloud_connection_id = data.ibm_pi_cloud_connection.cloud_connection.id
   pi_cidr                = var.machine_cidr
   pi_dns_server          = var.dns_server
+  pi_dhcp_snat_enabled   = var.enable_snat
   # the pi_dhcp_name param will be prefixed by the DHCP ID when created, so keep it short here:
   pi_dhcp_name = var.cluster_id
 }

data/data/powervs/cluster/power_network/variables.tf

@@ -40,3 +40,9 @@ variable "dns_server" {
   description = "The desired DNS server for the DHCP instance to server."
   default     = "1.1.1.1"
 }
+
+variable "enable_snat" {
+  type        = bool
+  description = "Boolean indicating if SNAT should be enabled."
+  default     = true
+}

data/data/powervs/variables-powervs.tf

@@ -54,12 +54,6 @@ variable "powervs_cos_instance_location" {
   default     = "global"
 }
 
-variable "powervs_cos_bucket_location" {
-  type        = string
-  description = "The location to create your COS bucket"
-  default     = "us-east"
-}
-
 variable "powervs_cos_storage_class" {
   type        = string
   description = "The plan used for your COS instance"
@@ -99,6 +93,11 @@ variable "powervs_vpc_subnet_name" {
   default     = ""
 }
 
+variable "powervs_enable_snat" {
+  type        = bool
+  description = "Boolean indicating if SNAT should be enabled or disabled."
+  default     = true
+}
 
 ################################################################
 # Configure instances

pkg/asset/cluster/tfvars.go

@@ -826,6 +826,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				CISInstanceCRN:       cisCRN,
 				DNSInstanceCRN:       dnsCRN,
 				PublishStrategy:      installConfig.Config.Publish,
+				EnableSNAT:           len(installConfig.Config.ImageContentSources) == 0,
 			},
 		)
 		if err != nil {

pkg/tfvars/powervs/powervs.go

@@ -38,6 +38,7 @@ type config struct {
 	ProcType             string `json:"powervs_proc_type"`
 	SysType              string `json:"powervs_sys_type"`
 	PublishStrategy      string `json:"powervs_publish_strategy"`
+	EnableSNAT           bool   `json:"powervs_enable_snat"`
 }
 
 // TFVarsSources contains the parameters to be converted into Terraform variables
@@ -57,6 +58,7 @@ type TFVarsSources struct {
 	VPCName              string
 	VPCSubnetName        string
 	PublishStrategy      types.PublishingStrategy
+	EnableSNAT           bool
 }
 
 // TFVars generates Power VS-specific Terraform variables launching the cluster.
@@ -116,6 +118,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		ProcType:             strings.ToLower(string(masterConfig.ProcessorType)),
 		SysType:              masterConfig.SystemType,
 		PublishStrategy:      string(sources.PublishStrategy),
+		EnableSNAT:           sources.EnableSNAT,
 	}
 	if masterConfig.Network.Name != nil {
 		cfg.NetworkName = *masterConfig.Network.Name

terraform/providers/ibm/go.mod

@@ -2,14 +2,14 @@ module github.com/openshift/installer/terraform/providers/ibm
 
 go 1.18
 
-require github.com/IBM-Cloud/terraform-provider-ibm v1.44.2
+require github.com/IBM-Cloud/terraform-provider-ibm v1.45.0
 
 require (
 	github.com/IBM-Cloud/bluemix-go v0.0.0-20220523145737-34645883de47 // indirect
-	github.com/IBM-Cloud/container-services-go-sdk v0.0.0-20220622142911-811d18c8c775 // indirect
-	github.com/IBM-Cloud/power-go-client v1.1.10 // indirect
+	github.com/IBM-Cloud/container-services-go-sdk v0.0.0-20220728135852-60ff775f7a8d // indirect
+	github.com/IBM-Cloud/power-go-client v1.1.12 // indirect
 	github.com/IBM/apigateway-go-sdk v0.0.0-20210714141226-a5d5d49caaca // indirect
-	github.com/IBM/appconfiguration-go-admin-sdk v0.2.3 // indirect
+	github.com/IBM/appconfiguration-go-admin-sdk v0.3.0 // indirect
 	github.com/IBM/appid-management-go-sdk v0.0.0-20210908164609-dd0e0eaf732f // indirect
 	github.com/IBM/cloud-databases-go-sdk v0.2.0 // indirect
 	github.com/IBM/cloudant-go-sdk v0.0.43 // indirect
@@ -24,10 +24,9 @@ require (
 	github.com/IBM/ibm-hpcs-tke-sdk v0.0.0-20211109141421-a4b61b05f7d1 // indirect
 	github.com/IBM/ibm-hpcs-uko-sdk v0.0.4 // indirect
 	github.com/IBM/keyprotect-go-client v0.7.0 // indirect
-	github.com/IBM/networking-go-sdk v0.31.0 // indirect
-	github.com/IBM/platform-services-go-sdk v0.26.1 // indirect
+	github.com/IBM/networking-go-sdk v0.33.0 // indirect
+	github.com/IBM/platform-services-go-sdk v0.28.2 // indirect
 	github.com/IBM/push-notifications-go-sdk v0.0.0-20210310100607-5790b96c47f5 // indirect
-	github.com/IBM/scc-go-sdk v1.3.4 // indirect
 	github.com/IBM/scc-go-sdk/v3 v3.1.6 // indirect
 	github.com/IBM/schematics-go-sdk v0.2.1 // indirect
 	github.com/IBM/secrets-manager-go-sdk v0.1.19 // indirect
@@ -127,7 +126,7 @@ require (
 	go.mongodb.org/mongo-driver v1.10.0 // indirect
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
-	golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f // indirect
+	golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect