Power VS: Create Remaining TF Resources

data/data/powervs/cluster/bootstrap/lb/main.tf

@@ -0,0 +1,53 @@
+# Using explicit depends_on as otherwise there are issues with updating and adding of pool members
+# Ref: https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/is_lb_listener
+resource "ibm_is_lb_pool_member" "machine_config_member" {
+  lb             = var.lb_int_id
+  pool           = var.machine_cfg_pool_id
+  port           = 22623
+  target_address = var.bootstrap_ip
+}
+
+resource "ibm_is_lb_pool_member" "api_member_int" {
+  depends_on     = [ibm_is_lb_pool_member.machine_config_member]
+  lb             = var.lb_int_id
+  pool           = var.api_pool_int_id
+  port           = 6443
+  target_address = var.bootstrap_ip
+}
+
+resource "ibm_is_lb_pool_member" "api_member" {
+  lb             = var.lb_ext_id
+  pool           = var.api_pool_ext_id
+  port           = 6443
+  target_address = var.bootstrap_ip
+}
+
+# bootstrap ssh pool, listener, member
+resource "ibm_is_lb_pool" "bootstrap_pool" {
+  depends_on     = [ibm_is_lb_pool_member.api_member]
+  name           = "bootstrap-node"
+  lb             = var.lb_ext_id
+  algorithm      = "round_robin"
+  protocol       = "tcp"
+  health_delay   = 5
+  health_retries = 2
+  health_timeout = 2
+  health_type    = "tcp"
+}
+
+# explicit depends because the LB will be in `UPDATE_PENDING` state and this will fail
+resource "ibm_is_lb_listener" "bootstrap_listener" {
+  depends_on   = [ibm_is_lb_pool_member.api_member]
+  lb           = var.lb_ext_id
+  port         = 22
+  protocol     = "tcp"
+  default_pool = ibm_is_lb_pool.bootstrap_pool.id
+}
+
+resource "ibm_is_lb_pool_member" "bootstrap" {
+  depends_on     = [ibm_is_lb_listener.bootstrap_listener]
+  lb             = var.lb_ext_id
+  pool           = ibm_is_lb_pool.bootstrap_pool.id
+  port           = 22
+  target_address = var.bootstrap_ip
+}

data/data/powervs/cluster/bootstrap/lb/ouputs.tf

@@ -0,0 +1,7 @@
+output api_member_ext_id {
+  value = ibm_is_lb_pool_member.api_member.id
+}
+
+output api_member_int_id {
+  value = ibm_is_lb_pool_member.api_member_int.id
+}

data/data/powervs/cluster/bootstrap/lb/variables.tf

@@ -0,0 +1,29 @@
+variable "bootstrap_ip" {
+  type        = string
+  description = "The IP address of the bootstrap node."
+}
+
+variable "lb_ext_id" {
+  type        = string
+  description = "The ID of the external load balancer."
+}
+
+variable "lb_int_id" {
+  type        = string
+  description = "The ID of the internal load balancer."
+}
+
+variable "machine_cfg_pool_id" {
+  type        = string
+  description = "The ID of the machine config load balancer pool."
+}
+
+variable "api_pool_int_id" {
+  type        = string
+  description = "The ID of the internal API load balancer pool."
+}
+
+variable "api_pool_ext_id" {
+  type        = string
+  description = "The ID of the external API load balancer pool."
+}

data/data/powervs/cluster/bootstrap/main.tf

@@ -1,77 +1,51 @@
-# TODO(mjturek): network and image data blocks can be in main module
-#                as master and bootstrap will be using the same
-#                network and image. Once we add in master module, make
-#                the move.
-data "ibm_pi_network" "network" {
-  pi_network_name      = var.network_name
-  pi_cloud_instance_id = var.cloud_instance_id
+provider "ibm" {
+  alias            = "vpc"
+  ibmcloud_api_key = var.api_key
+  region           = var.vpc_region
+  zone             = var.vpc_zone
 }
 
-data "ibm_resource_group" "cos_group" {
-  name = var.resource_group
+provider "ibm" {
+  alias            = "powervs"
+  ibmcloud_api_key = var.api_key
+  region           = var.powervs_region
+  zone             = var.powervs_zone
 }
 
-resource "ibm_resource_instance" "cos_instance" {
-  name              = "${var.cluster_id}-cos"
-  resource_group_id = data.ibm_resource_group.cos_group.id
-  service           = "cloud-object-storage"
-  plan              = "standard"
-  location          = var.cos_instance_location
-  tags              = [var.cluster_id]
-}
-
-# Create an IBM COS Bucket to store ignition
-resource "ibm_cos_bucket" "ignition" {
-  bucket_name          = "${var.cluster_id}-bootstrap-ign"
-  resource_instance_id = ibm_resource_instance.cos_instance.id
-  region_location      = var.cos_bucket_location
-  storage_class        = var.cos_storage_class
-}
-
-resource "ibm_resource_key" "cos_service_cred" {
-  name                 = "${var.cluster_id}-cred"
-  role                 = "Reader"
-  resource_instance_id = ibm_resource_instance.cos_instance.id
-  parameters           = { HMAC = true }
-}
-
-# Place the bootstrap ignition file in the ignition COS bucket
-resource "ibm_cos_bucket_object" "ignition" {
-  bucket_crn      = ibm_cos_bucket.ignition.crn
-  bucket_location = ibm_cos_bucket.ignition.region_location
-  content         = var.ignition
-  key             = "bootstrap.ign"
-  etag            = md5(var.ignition)
-}
-
-data "ibm_iam_auth_token" "iam_token" {}
-
-# Create the bootstrap instance
-resource "ibm_pi_instance" "bootstrap" {
-  pi_memory            = var.memory
-  pi_processors        = var.processors
-  pi_instance_name     = "${var.cluster_id}-bootstrap"
-  pi_proc_type         = var.proc_type
-  pi_image_id          = var.image_id
-  pi_sys_type          = var.sys_type
-  pi_cloud_instance_id = var.cloud_instance_id
-  pi_network {
-    network_id = data.ibm_pi_network.network.id
+module "vm" {
+  providers = {
+    ibm = ibm.powervs
   }
-  pi_user_data = base64encode(templatefile("${path.module}/templates/bootstrap.ign", {
-    HOSTNAME    = ibm_cos_bucket.ignition.s3_endpoint_public
-    BUCKET_NAME = ibm_cos_bucket.ignition.bucket_name
-    OBJECT_NAME = ibm_cos_bucket_object.ignition.key
-    IAM_TOKEN   = data.ibm_iam_auth_token.iam_token.iam_access_token
-  }))
-  pi_key_pair_name = var.key_id
-  pi_health_status = "WARNING"
-}
-
-data "ibm_pi_instance_ip" "bootstrap_ip" {
-  depends_on = [ibm_pi_instance.bootstrap]
+  source = "./vm"
+
+  resource_group        = var.resource_group
+  cluster_id            = var.cluster_id
+  ssh_key_id            = var.ssh_key_id
+  cos_bucket_location   = var.cos_bucket_location
+  cos_instance_location = var.cos_instance_location
+  cos_storage_class     = var.cos_storage_class
+  ignition              = var.ignition
+  memory                = var.memory
+  processors            = var.processors
+  proc_type             = var.proc_type
+  image_id              = var.image_id
+  sys_type              = var.sys_type
+  cloud_instance_id     = var.cloud_instance_id
+  dhcp_network_id       = var.dhcp_network_id
+  dhcp_id               = var.dhcp_id
+}
+
+module "lb" {
+  providers = {
+    ibm = ibm.vpc
+  }
+  source = "./lb"
 
-  pi_instance_name     = ibm_pi_instance.bootstrap.pi_instance_name
-  pi_network_name      = data.ibm_pi_network.network.pi_network_name
-  pi_cloud_instance_id = var.cloud_instance_id
+  bootstrap_ip        = module.vm.bootstrap_ip
+  lb_int_id           = var.lb_int_id
+  lb_ext_id           = var.lb_ext_id
+  machine_cfg_pool_id = var.machine_cfg_pool_id
+  api_pool_int_id     = var.api_pool_int_id
+  api_pool_ext_id     = var.api_pool_ext_id
 }
+

data/data/powervs/cluster/bootstrap/outputs.tf

@@ -1,3 +1,11 @@
-output "bootstrap_private_ip" {
-  value = data.ibm_pi_instance_ip.bootstrap_ip.ip
+output bootstrap_ip {
+  value = module.vm.bootstrap_ip
+}
+
+output api_member_ext_id {
+  value = module.lb.api_member_ext_id
+}
+
+output api_member_int_id {
+  value = module.lb.api_member_int_id
 }

data/data/powervs/cluster/bootstrap/variables.tf

@@ -67,3 +67,64 @@ variable "cos_storage_class" {
   type        = string
   description = "The storage class for the Cloud Object Store instance."
 }
+
+variable "vpc_region" {
+  type        = string
+  description = "The IBM Cloud region in which the VPC is created."
+}
+
+variable "vpc_zone" {
+  type        = string
+  description = "The IBM Cloud zone in which the VPC is created."
+}
+
+variable "vpc_id" {
+  type        = string
+  description = "The ID of the VPC created for the cluster load balancers."
+}
+
+variable "powervs_region" {
+  type        = string
+  description = "The Power VS region in which to create resources."
+}
+
+variable "powervs_zone" {
+  type        = string
+  description = "The Power VS zone in which to create resources."
+
+}
+
+variable "dhcp_id" {
+  type        = string
+  description = "The ID of the Power VS DHCP Service."
+}
+
+variable "dhcp_network_id" {
+  type        = string
+  description = "The ID of the Power VS DHCP network."
+}
+
+variable "lb_ext_id" {
+  type        = string
+  description = "The ID of the external load balancer in the IBM Cloud VPC"
+}
+
+variable "lb_int_id" {
+  type        = string
+  description = "The ID of the private load balancer in the IBM Cloud VPC"
+}
+
+variable "machine_cfg_pool_id" {
+  type        = string
+  description = "The ID of the load balancer pool for the machine-config server."
+}
+
+variable "api_pool_int_id" {
+  type        = string
+  description = "The ID of the private load balancer pool for the API server."
+}
+
+variable "api_pool_ext_id" {
+  type        = string
+  description = "The ID of the public load balancer pool for the API server."
+}

data/data/powervs/cluster/bootstrap/vm/main.tf

@@ -0,0 +1,76 @@
+data "ibm_resource_group" "cos_group" {
+  name = var.resource_group
+}
+
+resource "ibm_resource_instance" "cos_instance" {
+  name              = "${var.cluster_id}-cos"
+  resource_group_id = data.ibm_resource_group.cos_group.id
+  service           = "cloud-object-storage"
+  plan              = "standard"
+  location          = var.cos_instance_location
+  tags              = [var.cluster_id]
+}
+
+# Create an IBM COS Bucket to store ignition
+resource "ibm_cos_bucket" "ignition" {
+  bucket_name          = "${var.cluster_id}-bootstrap-ign"
+  resource_instance_id = ibm_resource_instance.cos_instance.id
+  region_location      = var.cos_bucket_location
+  storage_class        = var.cos_storage_class
+}
+
+resource "ibm_resource_key" "cos_service_cred" {
+  name                 = "${var.cluster_id}-cred"
+  role                 = "Reader"
+  resource_instance_id = ibm_resource_instance.cos_instance.id
+  parameters           = { HMAC = true }
+}
+
+# Place the bootstrap ignition file in the ignition COS bucket
+resource "ibm_cos_bucket_object" "ignition" {
+  bucket_crn      = ibm_cos_bucket.ignition.crn
+  bucket_location = ibm_cos_bucket.ignition.region_location
+  content         = var.ignition
+  key             = "bootstrap.ign"
+  etag            = md5(var.ignition)
+}
+
+data "ibm_iam_auth_token" "iam_token" {}
+
+# Create the bootstrap instance
+resource "ibm_pi_instance" "bootstrap" {
+  pi_memory            = var.memory
+  pi_processors        = var.processors
+  pi_instance_name     = "${var.cluster_id}-bootstrap"
+  pi_proc_type         = var.proc_type
+  pi_image_id          = var.image_id
+  pi_sys_type          = var.sys_type
+  pi_cloud_instance_id = var.cloud_instance_id
+  pi_network {
+    network_id = var.dhcp_network_id
+  }
+  pi_user_data = base64encode(templatefile("${path.module}/templates/bootstrap.ign", {
+    HOSTNAME    = ibm_cos_bucket.ignition.s3_endpoint_public
+    BUCKET_NAME = ibm_cos_bucket.ignition.bucket_name
+    OBJECT_NAME = ibm_cos_bucket_object.ignition.key
+    IAM_TOKEN   = data.ibm_iam_auth_token.iam_token.iam_access_token
+  }))
+  pi_key_pair_name = var.ssh_key_id
+  pi_health_status = "WARNING"
+}
+
+resource "time_sleep" "wait_for_bootstrap_macs" {
+  create_duration = "45s"
+
+  depends_on = [ibm_pi_instance.bootstrap]
+}
+
+locals {
+  bootstrap_ips = [for lease in data.ibm_pi_dhcp.dhcp_service_refresh.leases : lease.instance_ip if ibm_pi_instance.bootstrap.pi_network[0].mac_address == lease.instance_mac]
+}
+
+data "ibm_pi_dhcp" "dhcp_service_refresh" {
+  depends_on           = [time_sleep.wait_for_bootstrap_macs]
+  pi_cloud_instance_id = var.cloud_instance_id
+  pi_dhcp_id           = var.dhcp_id
+}

data/data/powervs/cluster/bootstrap/vm/outputs.tf

@@ -0,0 +1,4 @@
+output bootstrap_ip {
+  value = local.bootstrap_ips[0]
+}
+

data/data/powervs/cluster/bootstrap/vm/templates/bootstrap.ign

@@ -0,0 +1,16 @@
+{
+  "ignition": {
+    "version": "3.2.0",
+    "config": {
+      "replace": {
+        "source": "https://${HOSTNAME}/${BUCKET_NAME}/${OBJECT_NAME}",
+        "httpHeaders": [
+          {
+            "name": "Authorization",
+            "value": "${IAM_TOKEN}"
+          }
+        ]
+      }
+    }
+  }
+}