Rearranging TF stages so deletion of bootstrap stuff happens naturally

data/data/powervs/bootstrap-routing/main.tf

@@ -0,0 +1,39 @@
+locals {
+  api_servers       = var.powervs_expose_bootstrap ? concat(var.control_plane_ips, [var.bootstrap_private_ip]): var.control_plane_ips
+  api_servers_count = length(local.api_servers)
+}
+
+provider "ibm" {
+  ibmcloud_api_key = var.powervs_api_key
+  region           = var.powervs_vpc_region
+  zone             = var.powervs_vpc_zone
+}
+
+resource "ibm_is_lb_pool_member" "machine_config_member" {
+  count          = local.api_servers_count
+
+  lb             = var.load_balancer_int_id
+  pool           = var.machine_config_lb_pool
+  port           = 22623
+  target_address = local.api_servers[count.index]
+}
+
+resource "ibm_is_lb_pool_member" "api_member_int" {
+  depends_on     = [ibm_is_lb_pool_member.machine_config_member]
+  count          = local.api_servers_count
+
+  lb             = var.load_balancer_int_id
+  pool           = var.api_int_lb_pool
+  port           = 6443
+  target_address = local.api_servers[count.index]
+}
+
+resource "ibm_is_lb_pool_member" "api_member" {
+  depends_on     = [ibm_is_lb_pool_member.machine_config_member]
+  count          = local.api_servers_count
+
+  lb             = var.load_balancer_id
+  pool           = var.api_lb_pool
+  port           = 6443
+  target_address = local.api_servers[count.index]
+}

data/data/powervs/bootstrap-routing/variables.tf

@@ -0,0 +1,6 @@
+variable "load_balancer_id" { type = string }
+variable "load_balancer_int_id" { type = string }
+variable "machine_config_lb_pool" { type = string }
+variable "api_lb_pool" { type = string }
+variable "api_int_lb_pool" { type = string }
+variable "bootstrap_private_ip" { type = string }

data/data/powervs/bootstrap/lb/main.tf

@@ -0,0 +1,29 @@
+# bootstrap listener and backend pool
+resource "ibm_is_lb_listener" "bootstrap_listener" {
+  lb             = var.load_balancer_id
+  port           = 22
+  protocol       = "tcp"
+  default_pool   = ibm_is_lb_pool.bootstrap_pool.id
+}
+resource "ibm_is_lb_pool" "bootstrap_pool" {
+  #depends_on = [ibm_is_lb.load_balancer]
+
+  name           = "bootstrap-node"
+  lb             = var.load_balancer_id
+  algorithm      = "round_robin"
+  protocol       = "tcp"
+  health_delay   = 5
+  health_retries = 2
+  health_timeout = 2
+  health_type    = "tcp"
+}
+resource "ibm_is_lb_pool_member" "bootstrap" {
+  depends_on = [ibm_is_lb_listener.bootstrap_listener]
+
+  lb             = var.load_balancer_id
+  pool           = ibm_is_lb_pool.bootstrap_pool.id
+  port           = 22
+  target_address = var.bootstrap_private_ip
+}
+
+

data/data/powervs/bootstrap/lb/variables.tf

@@ -0,0 +1,2 @@
+variable "load_balancer_id" {}
+variable "bootstrap_private_ip" {}

data/data/powervs/bootstrap/main.tf

@@ -0,0 +1,44 @@
+provider "ibm" {
+  alias            = "vpc"
+  ibmcloud_api_key = var.powervs_api_key
+  region           = var.powervs_vpc_region
+  zone             = var.powervs_vpc_zone
+}
+
+provider "ibm" {
+  alias = "powervs"
+  ibmcloud_api_key = var.powervs_api_key
+  region           = var.powervs_region
+}
+
+module "vm" {
+  providers = {
+    ibm = ibm.powervs
+  }
+  source            = "./vm"
+
+  cloud_instance_id = var.powervs_cloud_instance_id
+  cluster_id        = var.cluster_id
+  cluster_key_id    = var.cluster_key_id
+  ignition_host     = var.bootstrap_ignition_host
+  ignition_bucket   = var.bootstrap_ignition_bucket
+  ignition_key      = var.bootstrap_ignition_key
+  boot_image        = var.boot_image_id
+
+  memory            = var.powervs_bootstrap_memory
+  processors        = var.powervs_bootstrap_processors
+  sys_type          = var.powervs_sys_type
+  proc_type         = var.powervs_proc_type
+  network_id        = var.network_id
+  network_name      = var.powervs_network_name
+}
+
+module "lb" {
+  providers = {
+    ibm = ibm.vpc
+  }
+  source = "./lb"
+
+  load_balancer_id     = var.load_balancer_id
+  bootstrap_private_ip = module.vm.bootstrap_private_ip
+}

data/data/powervs/bootstrap/outputs.tf

@@ -0,0 +1,3 @@
+output "bootstrap_private_ip" {
+  value = module.vm.bootstrap_private_ip
+}

data/data/powervs/bootstrap/variables.tf

@@ -0,0 +1,7 @@
+variable "bootstrap_ignition_host" { type = string }
+variable "bootstrap_ignition_bucket" { type = string }
+variable "bootstrap_ignition_key" { type = string }
+variable "cluster_key_id" { type = string }
+variable "boot_image_id" { type = string }
+variable "network_id" { type = string }
+variable "load_balancer_id" { type = string }

data/data/powervs/bootstrap/vm/main.tf

@@ -0,0 +1,32 @@
+data "ibm_iam_auth_token" "iam_token" {}
+
+# Create the bootstrap instance
+resource "ibm_pi_instance" "bootstrap" {
+  pi_memory            = var.memory
+  pi_processors        = var.processors
+  pi_instance_name     = "${var.cluster_id}-bootstrap"
+  pi_proc_type         = var.proc_type
+  pi_image_id          = var.boot_image
+  pi_sys_type          = var.sys_type
+  pi_cloud_instance_id = var.cloud_instance_id
+  pi_network {
+    network_id = var.network_id
+  }
+  pi_user_data         = base64encode(templatefile("${path.module}/templates/bootstrap.ign", {
+    HOSTNAME    = var.ignition_host
+    BUCKET_NAME = var.ignition_bucket
+    OBJECT_NAME = var.ignition_key
+    IAM_TOKEN   = data.ibm_iam_auth_token.iam_token.iam_access_token
+  }))
+  pi_key_pair_name     = var.cluster_key_id
+  pi_health_status     = "WARNING"
+}
+
+data "ibm_pi_instance_ip" "bootstrap_ip" {
+  depends_on = [ibm_pi_instance.bootstrap]
+
+  pi_instance_name     = ibm_pi_instance.bootstrap.pi_instance_name
+  pi_network_name      = var.network_name
+  pi_cloud_instance_id = var.cloud_instance_id
+}
+

data/data/powervs/bootstrap/vm/outputs.tf

@@ -0,0 +1,3 @@
+output "bootstrap_private_ip" {
+  value = data.ibm_pi_instance_ip.bootstrap_ip.ip
+}

data/data/powervs/bootstrap/vm/variables.tf

@@ -0,0 +1,14 @@
+variable "cloud_instance_id" {}
+variable "cluster_id" {}
+variable "cluster_key_id" {}
+variable "ignition_host" {}
+variable "ignition_bucket" {}
+variable "ignition_key" {}
+variable "boot_image" {}
+
+variable "memory" {}
+variable "processors" {}
+variable "proc_type" {}
+variable "sys_type" {}
+variable "network_id" {}
+variable "network_name" {}

data/data/powervs/cluster/bootstrap/main.tf

@@ -43,35 +43,3 @@ resource "ibm_cos_bucket_object" "ignition" {
   key             = "bootstrap.ign"
   etag            = md5(var.ignition)
 }
-
-data "ibm_iam_auth_token" "iam_token" {}
-
-# Create the bootstrap instance
-resource "ibm_pi_instance" "bootstrap" {
-  pi_memory            = var.memory
-  pi_processors        = var.processors
-  pi_instance_name     = "${var.cluster_id}-bootstrap"
-  pi_proc_type         = var.proc_type
-  pi_image_id          = var.image_id
-  pi_sys_type          = var.sys_type
-  pi_cloud_instance_id = var.cloud_instance_id
-  pi_network {
-    network_id = data.ibm_pi_network.network.id
-  }
-  pi_user_data         = base64encode(templatefile("${path.module}/templates/bootstrap.ign", {
-    HOSTNAME    = ibm_cos_bucket.ignition.s3_endpoint_public
-    BUCKET_NAME = ibm_cos_bucket.ignition.bucket_name
-    OBJECT_NAME = ibm_cos_bucket_object.ignition.key
-    IAM_TOKEN   = data.ibm_iam_auth_token.iam_token.iam_access_token
-  }))
-  pi_key_pair_name     = var.key_id
-  pi_health_status     = "WARNING"
-}
-
-data "ibm_pi_instance_ip" "bootstrap_ip" {
-  depends_on = [ibm_pi_instance.bootstrap]
-
-  pi_instance_name     = ibm_pi_instance.bootstrap.pi_instance_name
-  pi_network_name      = data.ibm_pi_network.network.pi_network_name
-  pi_cloud_instance_id = var.cloud_instance_id
-}

data/data/powervs/cluster/bootstrap/outputs.tf

@@ -1,3 +1,9 @@
-output "bootstrap_private_ip" {
-  value = data.ibm_pi_instance_ip.bootstrap_ip.ip
+output "bootstrap_ignition_host" {
+  value = ibm_cos_bucket.ignition.s3_endpoint_public
+}
+output "bootstrap_ignition_bucket" {
+  value = ibm_cos_bucket.ignition.bucket_name
+}
+output "bootstrap_ignition_key" {
+  value = ibm_cos_bucket_object.ignition.key
 }

data/data/powervs/cluster/bootstrap/variables.tf

@@ -4,12 +4,10 @@ variable "ignition" {}
 
 variable "cloud_instance_id" {}
 variable "resource_group" {}
-variable "image_id" {}
 variable "network_name" {}
 variable "proc_type" {}
 variable "sys_type" {}
 variable "cluster_id" {}
-variable "key_id" {}
 
 variable "cos_instance_location" {}
 variable "cos_bucket_location" {}

data/data/powervs/cluster/loadbalancer/alb.tf

@@ -1,6 +1,4 @@
 locals {
-  api_servers       = concat([var.bootstrap_ip], var.master_ips)
-  api_servers_count = length(var.master_ips) + 1 # bootstrap + master
   app_servers       = var.master_ips
   app_servers_count = length(var.master_ips)
 }
@@ -50,15 +48,6 @@ resource "ibm_is_lb_pool" "machine_config_pool" {
   health_timeout = 30
   health_type    = "tcp"
 }
-resource "ibm_is_lb_pool_member" "machine_config_member" {
-  depends_on = [ibm_is_lb_listener.machine_config_listener]
-  count      = local.api_servers_count
-
-  lb             = ibm_is_lb.load_balancer_int.id
-  pool           = ibm_is_lb_pool.machine_config_pool.id
-  port           = 22623
-  target_address = local.api_servers[count.index]
-}
 
 # api listener and backend pool (internal)
 resource "ibm_is_lb_listener" "api_listener_int" {
@@ -79,15 +68,6 @@ resource "ibm_is_lb_pool" "api_pool_int" {
   health_timeout = 30
   health_type    = "tcp"
 }
-resource "ibm_is_lb_pool_member" "api_member_int" {
-  depends_on = [ibm_is_lb_listener.api_listener_int, ibm_is_lb_pool_member.machine_config_member]
-  count      = local.api_servers_count
-
-  lb             = ibm_is_lb.load_balancer_int.id
-  pool           = ibm_is_lb_pool.api_pool_int.id
-  port           = 6443
-  target_address = local.api_servers[count.index]
-}
 
 # api listener and backend pool (external)
 resource "ibm_is_lb_listener" "api_listener" {
@@ -108,41 +88,3 @@ resource "ibm_is_lb_pool" "api_pool" {
   health_timeout = 30
   health_type    = "tcp"
 }
-resource "ibm_is_lb_pool_member" "api_member" {
-  depends_on = [ibm_is_lb_listener.api_listener, ibm_is_lb_pool_member.machine_config_member]
-  count      = local.api_servers_count
-
-  lb             = ibm_is_lb.load_balancer.id
-  pool           = ibm_is_lb_pool.api_pool.id
-  port           = 6443
-  target_address = local.api_servers[count.index]
-}
-
-# bootstrap listener and backend pool
-resource "ibm_is_lb_listener" "bootstrap_listener" {
-  lb             = ibm_is_lb.load_balancer.id
-  port           = 22
-  protocol       = "tcp"
-  default_pool   = ibm_is_lb_pool.bootstrap_pool.id
-}
-resource "ibm_is_lb_pool" "bootstrap_pool" {
-  depends_on = [ibm_is_lb.load_balancer]
-
-  name           = "bootstrap-node"
-  lb             = ibm_is_lb.load_balancer.id
-  algorithm      = "round_robin"
-  protocol       = "tcp"
-  health_delay   = 5
-  health_retries = 2
-  health_timeout = 2
-  health_type    = "tcp"
-}
-resource "ibm_is_lb_pool_member" "bootstrap" {
-  depends_on = [ibm_is_lb_listener.bootstrap_listener]
-
-  lb             = ibm_is_lb.load_balancer.id
-  pool           = ibm_is_lb_pool.bootstrap_pool.id
-  port           = 22
-  target_address = var.bootstrap_ip
-}
-

data/data/powervs/cluster/loadbalancer/outputs.tf

@@ -5,3 +5,23 @@ output "powervs_lb_hostname" {
 output "powervs_lb_int_hostname" {
   value = ibm_is_lb.load_balancer_int.hostname
 }
+
+output "load_balancer_id" {
+  value = ibm_is_lb.load_balancer.id
+}
+
+output "load_balancer_int_id" {
+  value = ibm_is_lb.load_balancer_int.id
+}
+
+output "machine_config_lb_pool" {
+  value = ibm_is_lb_pool.machine_config_pool.id
+}
+
+output "api_lb_pool" {
+  value = ibm_is_lb_pool.api_pool.id
+}
+
+output "api_int_lb_pool" {
+  value = ibm_is_lb_pool.api_pool_int.id
+}

data/data/powervs/cluster/loadbalancer/variables.tf

@@ -3,7 +3,6 @@ variable "cluster_id" {}
 variable "vpc_name" {}
 variable "vpc_subnet_id" {}
 
-variable "bootstrap_ip" {}
 variable "master_ips" {}
 
 variable "resource_group" {}

data/data/powervs/cluster/main.tf

@@ -36,8 +36,6 @@ module "bootstrap" {
   ignition     = var.ignition_bootstrap
   sys_type     = var.powervs_sys_type
   proc_type    = var.powervs_proc_type
-  key_id       = ibm_pi_key.cluster_key.key_id
-  image_id     = ibm_pi_image.boot_image.image_id
   network_name = var.powervs_network_name
 }
 
@@ -92,7 +90,6 @@ module "loadbalancer" {
   cluster_id     = var.cluster_id
   vpc_name       = var.powervs_vpc_name
   vpc_subnet_id  = data.ibm_is_subnet.vpc_subnet.id
-  bootstrap_ip   = module.bootstrap.bootstrap_private_ip
   master_ips     = module.master.master_ips
   resource_group = var.powervs_resource_group
 }

data/data/powervs/cluster/master/outputs.tf

@@ -1,3 +1,7 @@
 output "master_ips" {
   value = data.ibm_pi_instance_ip.master_ip.*.ip
 }
+
+output "network_id" {
+  value = data.ibm_pi_network.network.id
+}