From 64e2d79907ba281b8e93f74a4e514213117ef179 Mon Sep 17 00:00:00 2001 From: Hiro Miyamoto Date: Sat, 12 Mar 2022 14:30:58 -0500 Subject: [PATCH] Rearranging TF srages so deletion of bootstrap stuff happens naturally Signed-off-by: Hiro Miyamoto --- data/data/powervs/bootstrap-routing/main.tf | 39 +++++++++++++ .../powervs/bootstrap-routing/variables.tf | 6 ++ data/data/powervs/bootstrap/lb/main.tf | 29 ++++++++++ data/data/powervs/bootstrap/lb/variables.tf | 2 + data/data/powervs/bootstrap/main.tf | 44 ++++++++++++++ data/data/powervs/bootstrap/outputs.tf | 3 + data/data/powervs/bootstrap/variables.tf | 7 +++ data/data/powervs/bootstrap/vm/main.tf | 32 ++++++++++ data/data/powervs/bootstrap/vm/outputs.tf | 3 + .../vm}/templates/bootstrap.ign | 0 data/data/powervs/bootstrap/vm/variables.tf | 14 +++++ data/data/powervs/cluster/bootstrap/main.tf | 32 ---------- .../data/powervs/cluster/bootstrap/outputs.tf | 10 +++- .../powervs/cluster/bootstrap/variables.tf | 2 - data/data/powervs/cluster/loadbalancer/alb.tf | 58 ------------------- .../powervs/cluster/loadbalancer/outputs.tf | 20 +++++++ .../powervs/cluster/loadbalancer/variables.tf | 1 - data/data/powervs/cluster/main.tf | 3 - data/data/powervs/cluster/master/outputs.tf | 4 ++ data/data/powervs/cluster/outputs.tf | 44 ++++++++++++++ data/data/powervs/post-install/main.tf | 5 -- data/data/powervs/variables-powervs.tf | 6 ++ pkg/terraform/stages/powervs/stages.go | 24 +++++++- 23 files changed, 283 insertions(+), 105 deletions(-) create mode 100644 data/data/powervs/bootstrap-routing/main.tf create mode 100644 data/data/powervs/bootstrap-routing/variables.tf create mode 100644 data/data/powervs/bootstrap/lb/main.tf create mode 100644 data/data/powervs/bootstrap/lb/variables.tf create mode 100644 data/data/powervs/bootstrap/main.tf create mode 100644 data/data/powervs/bootstrap/outputs.tf create mode 100644 data/data/powervs/bootstrap/variables.tf create mode 100644 data/data/powervs/bootstrap/vm/main.tf create mode 100644 data/data/powervs/bootstrap/vm/outputs.tf rename data/data/powervs/{cluster/bootstrap => bootstrap/vm}/templates/bootstrap.ign (100%) create mode 100644 data/data/powervs/bootstrap/vm/variables.tf delete mode 100644 data/data/powervs/post-install/main.tf diff --git a/data/data/powervs/bootstrap-routing/main.tf b/data/data/powervs/bootstrap-routing/main.tf new file mode 100644 index 00000000000..5f4b02165a1 --- /dev/null +++ b/data/data/powervs/bootstrap-routing/main.tf @@ -0,0 +1,39 @@ +locals { + api_servers = var.powervs_expose_bootstrap ? concat(var.control_plane_ips, [var.bootstrap_private_ip]): var.control_plane_ips + api_servers_count = length(local.api_servers) +} + +provider "ibm" { + ibmcloud_api_key = var.powervs_api_key + region = var.powervs_vpc_region + zone = var.powervs_vpc_zone +} + +resource "ibm_is_lb_pool_member" "machine_config_member" { + count = local.api_servers_count + + lb = var.load_balancer_int_id + pool = var.machine_config_lb_pool + port = 22623 + target_address = local.api_servers[count.index] +} + +resource "ibm_is_lb_pool_member" "api_member_int" { + depends_on = [ibm_is_lb_pool_member.machine_config_member] + count = local.api_servers_count + + lb = var.load_balancer_int_id + pool = var.api_int_lb_pool + port = 6443 + target_address = local.api_servers[count.index] +} + +resource "ibm_is_lb_pool_member" "api_member" { + depends_on = [ibm_is_lb_pool_member.machine_config_member] + count = local.api_servers_count + + lb = var.load_balancer_id + pool = var.api_lb_pool + port = 6443 + target_address = local.api_servers[count.index] +} diff --git a/data/data/powervs/bootstrap-routing/variables.tf b/data/data/powervs/bootstrap-routing/variables.tf new file mode 100644 index 00000000000..5e2df26975c --- /dev/null +++ b/data/data/powervs/bootstrap-routing/variables.tf @@ -0,0 +1,6 @@ +variable "load_balancer_id" { type = string } +variable "load_balancer_int_id" { type = string } +variable "machine_config_lb_pool" { type = string } +variable "api_lb_pool" { type = string } +variable "api_int_lb_pool" { type = string } +variable "bootstrap_private_ip" { type = string } diff --git a/data/data/powervs/bootstrap/lb/main.tf b/data/data/powervs/bootstrap/lb/main.tf new file mode 100644 index 00000000000..4a3bea58aa4 --- /dev/null +++ b/data/data/powervs/bootstrap/lb/main.tf @@ -0,0 +1,29 @@ +# bootstrap listener and backend pool +resource "ibm_is_lb_listener" "bootstrap_listener" { + lb = var.load_balancer_id + port = 22 + protocol = "tcp" + default_pool = ibm_is_lb_pool.bootstrap_pool.id +} +resource "ibm_is_lb_pool" "bootstrap_pool" { + #depends_on = [ibm_is_lb.load_balancer] + + name = "bootstrap-node" + lb = var.load_balancer_id + algorithm = "round_robin" + protocol = "tcp" + health_delay = 5 + health_retries = 2 + health_timeout = 2 + health_type = "tcp" +} +resource "ibm_is_lb_pool_member" "bootstrap" { + depends_on = [ibm_is_lb_listener.bootstrap_listener] + + lb = var.load_balancer_id + pool = ibm_is_lb_pool.bootstrap_pool.id + port = 22 + target_address = var.bootstrap_private_ip +} + + diff --git a/data/data/powervs/bootstrap/lb/variables.tf b/data/data/powervs/bootstrap/lb/variables.tf new file mode 100644 index 00000000000..21b79f9532c --- /dev/null +++ b/data/data/powervs/bootstrap/lb/variables.tf @@ -0,0 +1,2 @@ +variable "load_balancer_id" {} +variable "bootstrap_private_ip" {} diff --git a/data/data/powervs/bootstrap/main.tf b/data/data/powervs/bootstrap/main.tf new file mode 100644 index 00000000000..8715fe3e9d9 --- /dev/null +++ b/data/data/powervs/bootstrap/main.tf @@ -0,0 +1,44 @@ +provider "ibm" { + alias = "vpc" + ibmcloud_api_key = var.powervs_api_key + region = var.powervs_vpc_region + zone = var.powervs_vpc_zone +} + +provider "ibm" { + alias = "powervs" + ibmcloud_api_key = var.powervs_api_key + region = var.powervs_region +} + +module "vm" { + providers = { + ibm = ibm.powervs + } + source = "./vm" + + cloud_instance_id = var.powervs_cloud_instance_id + cluster_id = var.cluster_id + cluster_key_id = var.cluster_key_id + ignition_host = var.bootstrap_ignition_host + ignition_bucket = var.bootstrap_ignition_bucket + ignition_key = var.bootstrap_ignition_key + boot_image = var.boot_image_id + + memory = var.powervs_bootstrap_memory + processors = var.powervs_bootstrap_processors + sys_type = var.powervs_sys_type + proc_type = var.powervs_proc_type + network_id = var.network_id + network_name = var.powervs_network_name +} + +module "lb" { + providers = { + ibm = ibm.vpc + } + source = "./lb" + + load_balancer_id = var.load_balancer_id + bootstrap_private_ip = module.vm.bootstrap_private_ip +} diff --git a/data/data/powervs/bootstrap/outputs.tf b/data/data/powervs/bootstrap/outputs.tf new file mode 100644 index 00000000000..e582d6f578e --- /dev/null +++ b/data/data/powervs/bootstrap/outputs.tf @@ -0,0 +1,3 @@ +output "bootstrap_private_ip" { + value = module.vm.bootstrap_private_ip +} diff --git a/data/data/powervs/bootstrap/variables.tf b/data/data/powervs/bootstrap/variables.tf new file mode 100644 index 00000000000..a833f05a875 --- /dev/null +++ b/data/data/powervs/bootstrap/variables.tf @@ -0,0 +1,7 @@ +variable "bootstrap_ignition_host" { type = string } +variable "bootstrap_ignition_bucket" { type = string } +variable "bootstrap_ignition_key" { type = string } +variable "cluster_key_id" { type = string } +variable "boot_image_id" { type = string } +variable "network_id" { type = string } +variable "load_balancer_id" { type = string } diff --git a/data/data/powervs/bootstrap/vm/main.tf b/data/data/powervs/bootstrap/vm/main.tf new file mode 100644 index 00000000000..37b327cd9cc --- /dev/null +++ b/data/data/powervs/bootstrap/vm/main.tf @@ -0,0 +1,32 @@ +data "ibm_iam_auth_token" "iam_token" {} + +# Create the bootstrap instance +resource "ibm_pi_instance" "bootstrap" { + pi_memory = var.memory + pi_processors = var.processors + pi_instance_name = "${var.cluster_id}-bootstrap" + pi_proc_type = var.proc_type + pi_image_id = var.boot_image + pi_sys_type = var.sys_type + pi_cloud_instance_id = var.cloud_instance_id + pi_network { + network_id = var.network_id + } + pi_user_data = base64encode(templatefile("${path.module}/templates/bootstrap.ign", { + HOSTNAME = var.ignition_host + BUCKET_NAME = var.ignition_bucket + OBJECT_NAME = var.ignition_key + IAM_TOKEN = data.ibm_iam_auth_token.iam_token.iam_access_token + })) + pi_key_pair_name = var.cluster_key_id + pi_health_status = "WARNING" +} + +data "ibm_pi_instance_ip" "bootstrap_ip" { + depends_on = [ibm_pi_instance.bootstrap] + + pi_instance_name = ibm_pi_instance.bootstrap.pi_instance_name + pi_network_name = var.network_name + pi_cloud_instance_id = var.cloud_instance_id +} + diff --git a/data/data/powervs/bootstrap/vm/outputs.tf b/data/data/powervs/bootstrap/vm/outputs.tf new file mode 100644 index 00000000000..1fdff08d8b9 --- /dev/null +++ b/data/data/powervs/bootstrap/vm/outputs.tf @@ -0,0 +1,3 @@ +output "bootstrap_private_ip" { + value = data.ibm_pi_instance_ip.bootstrap_ip.ip +} diff --git a/data/data/powervs/cluster/bootstrap/templates/bootstrap.ign b/data/data/powervs/bootstrap/vm/templates/bootstrap.ign similarity index 100% rename from data/data/powervs/cluster/bootstrap/templates/bootstrap.ign rename to data/data/powervs/bootstrap/vm/templates/bootstrap.ign diff --git a/data/data/powervs/bootstrap/vm/variables.tf b/data/data/powervs/bootstrap/vm/variables.tf new file mode 100644 index 00000000000..52c5f351ea9 --- /dev/null +++ b/data/data/powervs/bootstrap/vm/variables.tf @@ -0,0 +1,14 @@ +variable "cloud_instance_id" {} +variable "cluster_id" {} +variable "cluster_key_id" {} +variable "ignition_host" {} +variable "ignition_bucket" {} +variable "ignition_key" {} +variable "boot_image" {} + +variable "memory" {} +variable "processors" {} +variable "proc_type" {} +variable "sys_type" {} +variable "network_id" {} +variable "network_name" {} diff --git a/data/data/powervs/cluster/bootstrap/main.tf b/data/data/powervs/cluster/bootstrap/main.tf index e528e1cc4ec..e5352f68131 100644 --- a/data/data/powervs/cluster/bootstrap/main.tf +++ b/data/data/powervs/cluster/bootstrap/main.tf @@ -43,35 +43,3 @@ resource "ibm_cos_bucket_object" "ignition" { key = "bootstrap.ign" etag = md5(var.ignition) } - -data "ibm_iam_auth_token" "iam_token" {} - -# Create the bootstrap instance -resource "ibm_pi_instance" "bootstrap" { - pi_memory = var.memory - pi_processors = var.processors - pi_instance_name = "${var.cluster_id}-bootstrap" - pi_proc_type = var.proc_type - pi_image_id = var.image_id - pi_sys_type = var.sys_type - pi_cloud_instance_id = var.cloud_instance_id - pi_network { - network_id = data.ibm_pi_network.network.id - } - pi_user_data = base64encode(templatefile("${path.module}/templates/bootstrap.ign", { - HOSTNAME = ibm_cos_bucket.ignition.s3_endpoint_public - BUCKET_NAME = ibm_cos_bucket.ignition.bucket_name - OBJECT_NAME = ibm_cos_bucket_object.ignition.key - IAM_TOKEN = data.ibm_iam_auth_token.iam_token.iam_access_token - })) - pi_key_pair_name = var.key_id - pi_health_status = "WARNING" -} - -data "ibm_pi_instance_ip" "bootstrap_ip" { - depends_on = [ibm_pi_instance.bootstrap] - - pi_instance_name = ibm_pi_instance.bootstrap.pi_instance_name - pi_network_name = data.ibm_pi_network.network.pi_network_name - pi_cloud_instance_id = var.cloud_instance_id -} diff --git a/data/data/powervs/cluster/bootstrap/outputs.tf b/data/data/powervs/cluster/bootstrap/outputs.tf index 1fdff08d8b9..c545cb44a43 100644 --- a/data/data/powervs/cluster/bootstrap/outputs.tf +++ b/data/data/powervs/cluster/bootstrap/outputs.tf @@ -1,3 +1,9 @@ -output "bootstrap_private_ip" { - value = data.ibm_pi_instance_ip.bootstrap_ip.ip +output "bootstrap_ignition_host" { + value = ibm_cos_bucket.ignition.s3_endpoint_public +} +output "bootstrap_ignition_bucket" { + value = ibm_cos_bucket.ignition.bucket_name +} +output "bootstrap_ignition_key" { + value = ibm_cos_bucket_object.ignition.key } diff --git a/data/data/powervs/cluster/bootstrap/variables.tf b/data/data/powervs/cluster/bootstrap/variables.tf index b5dc4531368..7dc5ca7755f 100644 --- a/data/data/powervs/cluster/bootstrap/variables.tf +++ b/data/data/powervs/cluster/bootstrap/variables.tf @@ -4,12 +4,10 @@ variable "ignition" {} variable "cloud_instance_id" {} variable "resource_group" {} -variable "image_id" {} variable "network_name" {} variable "proc_type" {} variable "sys_type" {} variable "cluster_id" {} -variable "key_id" {} variable "cos_instance_location" {} variable "cos_bucket_location" {} diff --git a/data/data/powervs/cluster/loadbalancer/alb.tf b/data/data/powervs/cluster/loadbalancer/alb.tf index 4b3f126cda0..9f7adfa0106 100644 --- a/data/data/powervs/cluster/loadbalancer/alb.tf +++ b/data/data/powervs/cluster/loadbalancer/alb.tf @@ -1,6 +1,4 @@ locals { - api_servers = concat([var.bootstrap_ip], var.master_ips) - api_servers_count = length(var.master_ips) + 1 # bootstrap + master app_servers = var.master_ips app_servers_count = length(var.master_ips) } @@ -50,15 +48,6 @@ resource "ibm_is_lb_pool" "machine_config_pool" { health_timeout = 30 health_type = "tcp" } -resource "ibm_is_lb_pool_member" "machine_config_member" { - depends_on = [ibm_is_lb_listener.machine_config_listener] - count = local.api_servers_count - - lb = ibm_is_lb.load_balancer_int.id - pool = ibm_is_lb_pool.machine_config_pool.id - port = 22623 - target_address = local.api_servers[count.index] -} # api listener and backend pool (internal) resource "ibm_is_lb_listener" "api_listener_int" { @@ -79,15 +68,6 @@ resource "ibm_is_lb_pool" "api_pool_int" { health_timeout = 30 health_type = "tcp" } -resource "ibm_is_lb_pool_member" "api_member_int" { - depends_on = [ibm_is_lb_listener.api_listener_int, ibm_is_lb_pool_member.machine_config_member] - count = local.api_servers_count - - lb = ibm_is_lb.load_balancer_int.id - pool = ibm_is_lb_pool.api_pool_int.id - port = 6443 - target_address = local.api_servers[count.index] -} # api listener and backend pool (external) resource "ibm_is_lb_listener" "api_listener" { @@ -108,41 +88,3 @@ resource "ibm_is_lb_pool" "api_pool" { health_timeout = 30 health_type = "tcp" } -resource "ibm_is_lb_pool_member" "api_member" { - depends_on = [ibm_is_lb_listener.api_listener, ibm_is_lb_pool_member.machine_config_member] - count = local.api_servers_count - - lb = ibm_is_lb.load_balancer.id - pool = ibm_is_lb_pool.api_pool.id - port = 6443 - target_address = local.api_servers[count.index] -} - -# bootstrap listener and backend pool -resource "ibm_is_lb_listener" "bootstrap_listener" { - lb = ibm_is_lb.load_balancer.id - port = 22 - protocol = "tcp" - default_pool = ibm_is_lb_pool.bootstrap_pool.id -} -resource "ibm_is_lb_pool" "bootstrap_pool" { - depends_on = [ibm_is_lb.load_balancer] - - name = "bootstrap-node" - lb = ibm_is_lb.load_balancer.id - algorithm = "round_robin" - protocol = "tcp" - health_delay = 5 - health_retries = 2 - health_timeout = 2 - health_type = "tcp" -} -resource "ibm_is_lb_pool_member" "bootstrap" { - depends_on = [ibm_is_lb_listener.bootstrap_listener] - - lb = ibm_is_lb.load_balancer.id - pool = ibm_is_lb_pool.bootstrap_pool.id - port = 22 - target_address = var.bootstrap_ip -} - diff --git a/data/data/powervs/cluster/loadbalancer/outputs.tf b/data/data/powervs/cluster/loadbalancer/outputs.tf index b4c6c184fb8..50e3f74fd7c 100644 --- a/data/data/powervs/cluster/loadbalancer/outputs.tf +++ b/data/data/powervs/cluster/loadbalancer/outputs.tf @@ -5,3 +5,23 @@ output "powervs_lb_hostname" { output "powervs_lb_int_hostname" { value = ibm_is_lb.load_balancer_int.hostname } + +output "load_balancer_id" { + value = ibm_is_lb.load_balancer.id +} + +output "load_balancer_int_id" { + value = ibm_is_lb.load_balancer_int.id +} + +output "machine_config_lb_pool" { + value = ibm_is_lb_pool.machine_config_pool.id +} + +output "api_lb_pool" { + value = ibm_is_lb_pool.api_pool.id +} + +output "api_int_lb_pool" { + value = ibm_is_lb_pool.api_pool_int.id +} diff --git a/data/data/powervs/cluster/loadbalancer/variables.tf b/data/data/powervs/cluster/loadbalancer/variables.tf index c64b3344fa4..030a5d55cc1 100644 --- a/data/data/powervs/cluster/loadbalancer/variables.tf +++ b/data/data/powervs/cluster/loadbalancer/variables.tf @@ -3,7 +3,6 @@ variable "cluster_id" {} variable "vpc_name" {} variable "vpc_subnet_id" {} -variable "bootstrap_ip" {} variable "master_ips" {} variable "resource_group" {} diff --git a/data/data/powervs/cluster/main.tf b/data/data/powervs/cluster/main.tf index 620d9611ec5..80d37db126c 100644 --- a/data/data/powervs/cluster/main.tf +++ b/data/data/powervs/cluster/main.tf @@ -36,8 +36,6 @@ module "bootstrap" { ignition = var.ignition_bootstrap sys_type = var.powervs_sys_type proc_type = var.powervs_proc_type - key_id = ibm_pi_key.cluster_key.key_id - image_id = ibm_pi_image.boot_image.image_id network_name = var.powervs_network_name } @@ -92,7 +90,6 @@ module "loadbalancer" { cluster_id = var.cluster_id vpc_name = var.powervs_vpc_name vpc_subnet_id = data.ibm_is_subnet.vpc_subnet.id - bootstrap_ip = module.bootstrap.bootstrap_private_ip master_ips = module.master.master_ips resource_group = var.powervs_resource_group } diff --git a/data/data/powervs/cluster/master/outputs.tf b/data/data/powervs/cluster/master/outputs.tf index 910e18ebb0b..9dbb875aacb 100644 --- a/data/data/powervs/cluster/master/outputs.tf +++ b/data/data/powervs/cluster/master/outputs.tf @@ -1,3 +1,7 @@ output "master_ips" { value = data.ibm_pi_instance_ip.master_ip.*.ip } + +output "network_id" { + value = data.ibm_pi_network.network.id +} diff --git a/data/data/powervs/cluster/outputs.tf b/data/data/powervs/cluster/outputs.tf index 7c355853f60..e98fc1ada81 100644 --- a/data/data/powervs/cluster/outputs.tf +++ b/data/data/powervs/cluster/outputs.tf @@ -5,3 +5,47 @@ output "bootstrap_ip" { output "control_plane_ips" { value = module.master.master_ips } + +output "cluster_key_id" { + value = ibm_pi_key.cluster_key.key_id +} + +output "bootstrap_ignition_host" { + value = module.bootstrap.bootstrap_ignition_host +} + +output "bootstrap_ignition_bucket" { + value = module.bootstrap.bootstrap_ignition_bucket +} + +output "bootstrap_ignition_key" { + value = module.bootstrap.bootstrap_ignition_key +} + +output "boot_image_id" { + value = ibm_pi_image.boot_image.image_id +} + +output "network_id" { + value = module.master.network_id +} + +output "load_balancer_id" { + value = module.loadbalancer.load_balancer_id +} + +output "load_balancer_int_id" { + value = module.loadbalancer.load_balancer_int_id +} + +output "machine_config_lb_pool" { + value = module.loadbalancer.machine_config_lb_pool +} + +output "api_lb_pool" { + value = module.loadbalancer.api_lb_pool +} + +output "api_int_lb_pool" { + value = module.loadbalancer.api_int_lb_pool +} diff --git a/data/data/powervs/post-install/main.tf b/data/data/powervs/post-install/main.tf deleted file mode 100644 index bda13b12ade..00000000000 --- a/data/data/powervs/post-install/main.tf +++ /dev/null @@ -1,5 +0,0 @@ -provider "ibm" { - alias = "powervs" - ibmcloud_api_key = var.powervs_api_key - region = var.powervs_region -} diff --git a/data/data/powervs/variables-powervs.tf b/data/data/powervs/variables-powervs.tf index 21b0001bee5..722df0892ab 100644 --- a/data/data/powervs/variables-powervs.tf +++ b/data/data/powervs/variables-powervs.tf @@ -140,6 +140,12 @@ variable "powervs_vpc_subnet_name" { description = "Name of the VPC subnet connected via DirectLink to the Power VS private network." } +variable "powervs_expose_bootstrap" { + type = bool + description = "Setting this to false allows the bootstrap resources to be removed from the cluster load balancers." + default = true +} + ################################################################ # Configure DNS ################################################################ diff --git a/pkg/terraform/stages/powervs/stages.go b/pkg/terraform/stages/powervs/stages.go index 6c86ce04caa..24b2339da1f 100644 --- a/pkg/terraform/stages/powervs/stages.go +++ b/pkg/terraform/stages/powervs/stages.go @@ -1,9 +1,12 @@ package powervs import ( + "github.com/hashicorp/terraform-exec/tfexec" "github.com/openshift/installer/pkg/terraform" "github.com/openshift/installer/pkg/terraform/providers" "github.com/openshift/installer/pkg/terraform/stages" + powervstypes "github.com/openshift/installer/pkg/types/powervs" + "github.com/pkg/errors" ) // PlatformStages are the stages to run to provision the infrastructure in PowerVS. @@ -12,6 +15,23 @@ var PlatformStages = []terraform.Stage{ "cluster", []providers.Provider{providers.IBM, providers.Ignition}), stages.NewStage("powervs", - "post-install", - []providers.Provider{providers.IBM}), + "bootstrap", + []providers.Provider{providers.IBM, providers.Ignition}, + stages.WithNormalBootstrapDestroy()), + stages.NewStage("powervs", + "bootstrap-routing", + []providers.Provider{providers.IBM}, + stages.WithCustomBootstrapDestroy(removeFromLoadBalancers)), +} + +func removeFromLoadBalancers(s stages.SplitStage, directory string, varFiles []string) error { + opts := make([]tfexec.ApplyOption, 0, len(varFiles)+1) + for _, varFile := range varFiles { + opts = append(opts, tfexec.VarFile(varFile)) + } + opts = append(opts, tfexec.Var("powervs_expose_bootstrap=false")) + return errors.Wrap( + terraform.Apply(directory, powervstypes.Name, s, opts...), + "failed disabling bootstrap load balancing", + ) }