AlibabaCloud: fix for bootstrap teardown removing slb backends

data/data/alibabacloud/bootstrap/main.tf

@@ -155,12 +155,3 @@ resource "alicloud_instance" "bootstrap" {
   )
 }
 
-resource "alicloud_slb_backend_server" "slb_attachment_bootstraps" {
-  count = length(var.slb_ids)
-
-  load_balancer_id = var.slb_ids[count.index]
-  backend_servers {
-    server_id = alicloud_instance.bootstrap.id
-    weight    = 90
-  }
-}

data/data/alibabacloud/bootstrap/outputs.tf

@@ -1,3 +1,7 @@
 output "bootstrap_ip" {
   value = local.is_external ? data.alicloud_instances.bootstrap_data.instances.0.public_ip : data.alicloud_instances.bootstrap_data.instances.0.private_ip
 }
+
+output "bootstrap_ecs_id" {
+  value = alicloud_instance.bootstrap.id
+}

data/data/alibabacloud/cluster/master/main.tf

@@ -34,12 +34,3 @@ resource "alicloud_instance" "master" {
     var.tags,
   )
 }
-
-resource "alicloud_slb_backend_server" "slb_attachment_masters" {
-  count            = "${var.slb_group_length * length(alicloud_instance.master.*.id)}"
-  load_balancer_id = "${element(var.slb_ids, ceil(count.index / length(alicloud_instance.master.*.id)))}"
-  backend_servers {
-    server_id = "${element(alicloud_instance.master.*.id, count.index)}"
-    weight    = 90
-  }
-}

data/data/alibabacloud/cluster/outputs.tf

@@ -21,3 +21,7 @@ output "sg_master_id" {
 output "control_plane_ips" {
   value = module.master.master_ecs_private_ips
 }
+
+output "master_ecs_ids" {
+  value = module.master.master_ecs_ids
+}

data/data/alibabacloud/slb_attach_controlplane/main.tf

@@ -0,0 +1,26 @@
+provider "alicloud" {
+  access_key = var.ali_access_key
+  secret_key = var.ali_secret_key
+  region     = var.ali_region_id
+}
+
+/*
+This attachment will be called after the teardown of the bootstrap stage occurs. There is an issue
+that occurs during the boostrap teardown that removes all backend servers from the SLB. This adds controlplane
+servers into the the list of backend servers for the SLB. The ali_bootstrap_lb defaults to true and when cleanup
+is called we set ali_bootstrap_lb to false so that it is removed from the SLB backend.
+*/
+resource "alicloud_slb_backend_server" "slb_attach_controlplane" {
+  count            = length(var.slb_ids)
+  load_balancer_id = var.slb_ids[count.index]
+
+  dynamic "backend_servers" {
+
+    for_each = var.ali_bootstrap_lb ? concat([var.bootstrap_ecs_id], var.master_ecs_ids) : var.master_ecs_ids
+
+    content {
+      server_id = backend_servers.value
+      weight    = 90
+    }
+  }
+}

data/data/alibabacloud/slb_attach_controlplane/variables.tf

@@ -0,0 +1,14 @@
+variable "slb_ids" {
+  type        = list(string)
+  description = "The IDs of the load balancers to which to attach the bootstrap and control plane VMs."
+}
+
+variable "master_ecs_ids" {
+  type        = list(string)
+  description = "The list of control plane instance ids."
+}
+
+variable "bootstrap_ecs_id" {
+  type        = string
+  description = "The ID for the bootstrap instance."
+}

data/data/alibabacloud/variables-alibabacloud.tf

@@ -94,3 +94,9 @@ The stub Ignition configuration used to boot the bootstrap ECS instance. This al
 specified in ‘ali_ignition_bucket’.
 EOF
 }
+
+variable "ali_bootstrap_lb" {
+  type        = bool
+  description = "Setting this to false allows the bootstrap resources to be removed from the cluster load balancers."
+  default     = true
+}

pkg/terraform/stages/alibabacloud/stages.go

@@ -1,9 +1,13 @@
 package alibabacloud
 
 import (
+	"github.com/hashicorp/terraform-exec/tfexec"
+	"github.com/pkg/errors"
+
 	"github.com/openshift/installer/pkg/terraform"
 	"github.com/openshift/installer/pkg/terraform/providers"
 	"github.com/openshift/installer/pkg/terraform/stages"
+	alitypes "github.com/openshift/installer/pkg/types/alibabacloud"
 )
 
 // PlatformStages are the stages to run to provision the infrastructure in Alibaba Cloud.
@@ -19,4 +23,27 @@ var PlatformStages = []terraform.Stage{
 		[]providers.Provider{providers.AliCloud},
 		stages.WithNormalBootstrapDestroy(),
 	),
+	// This stage has been added due to an issue that occurs when the bootstrap stage performs
+	// the teardown of the bootstrap assets. The teardown of the bootstrap SLB attachment causes
+	// all of the controlplane backends to be removed from the SLB. This stage attaches the
+	// master VMs and the bootstrap VMs as backend servers to the SLBs on create. Later,
+	// on bootstrap destroy, this stages removes only the bootstrap VM from the backend servers.
+	stages.NewStage(
+		"alibabacloud",
+		"slb_attach_controlplane",
+		[]providers.Provider{providers.AliCloud},
+		stages.WithCustomBootstrapDestroy(removeFromLoadBalancers),
+	),
+}
+
+func removeFromLoadBalancers(s stages.SplitStage, directory string, varFiles []string) error {
+	opts := make([]tfexec.ApplyOption, 0, len(varFiles)+1)
+	for _, varFile := range varFiles {
+		opts = append(opts, tfexec.VarFile(varFile))
+	}
+	opts = append(opts, tfexec.Var("ali_bootstrap_lb=false"))
+	return errors.Wrap(
+		terraform.Apply(directory, alitypes.Name, s, opts...),
+		"failed disabling bootstrap load balancing",
+	)
 }