From f80acc2f33e484b0d0c5af0cddcc209686e1a1be Mon Sep 17 00:00:00 2001 From: John Hixson Date: Wed, 23 Jun 2021 13:44:39 -0700 Subject: [PATCH] Azure: Split terraform into stages vnet: create permanent network resources bootstrap: create bootstrap resources cluster: create permanent cluster resources This is needed for Terraform v0.14.6 and later since Terraform deletions now destroy all resource dependencies. https://issues.redhat.com/browse/CORS-1695 --- data/data/azure/bootstrap/main.tf | 47 ++-- data/data/azure/bootstrap/outputs.tf | 3 + data/data/azure/bootstrap/variables.tf | 89 +++----- data/data/azure/{ => cluster}/dns/dns.tf | 0 .../data/azure/{ => cluster}/dns/variables.tf | 0 data/data/azure/{ => cluster}/dns/versions.tf | 0 data/data/azure/cluster/main.tf | 73 +++++++ .../data/azure/{ => cluster}/master/master.tf | 0 data/data/azure/cluster/master/outputs.tf | 3 + .../azure/{ => cluster}/master/variables.tf | 0 .../azure/{ => cluster}/master/versions.tf | 0 data/data/azure/cluster/outputs.tf | 3 + data/data/azure/cluster/variables.tf | 94 ++++++++ data/data/azure/{ => cluster}/versions.tf | 0 data/data/azure/main.tf | 201 ------------------ data/data/azure/master/outputs.tf | 0 data/data/azure/vnet/common.tf | 40 ++-- data/data/azure/vnet/internal-lb.tf | 20 +- data/data/azure/vnet/main.tf | 104 +++++++++ data/data/azure/vnet/nsg.tf | 10 +- data/data/azure/vnet/outputs.tf | 38 +++- data/data/azure/vnet/public-lb.tf | 48 ++--- data/data/azure/vnet/variables.tf | 83 -------- data/data/azure/vnet/vnet.tf | 22 +- pkg/terraform/stages/azure/stages.go | 13 ++ pkg/terraform/stages/compat/stage.go | 11 - pkg/terraform/stages/platform/stages.go | 4 + 27 files changed, 452 insertions(+), 454 deletions(-) create mode 100644 data/data/azure/bootstrap/outputs.tf rename data/data/azure/{ => cluster}/dns/dns.tf (100%) rename data/data/azure/{ => cluster}/dns/variables.tf (100%) rename data/data/azure/{ => cluster}/dns/versions.tf (100%) create mode 100644 data/data/azure/cluster/main.tf rename data/data/azure/{ => cluster}/master/master.tf (100%) create mode 100644 data/data/azure/cluster/master/outputs.tf rename data/data/azure/{ => cluster}/master/variables.tf (100%) rename data/data/azure/{ => cluster}/master/versions.tf (100%) create mode 100644 data/data/azure/cluster/outputs.tf create mode 100644 data/data/azure/cluster/variables.tf rename data/data/azure/{ => cluster}/versions.tf (100%) delete mode 100644 data/data/azure/main.tf delete mode 100644 data/data/azure/master/outputs.tf create mode 100644 data/data/azure/vnet/main.tf delete mode 100644 data/data/azure/vnet/variables.tf create mode 100644 pkg/terraform/stages/azure/stages.go diff --git a/data/data/azure/bootstrap/main.tf b/data/data/azure/bootstrap/main.tf index 28634c8eba7..11731b04385 100644 --- a/data/data/azure/bootstrap/main.tf +++ b/data/data/azure/bootstrap/main.tf @@ -2,6 +2,21 @@ locals { bootstrap_nic_ip_v4_configuration_name = "bootstrap-nic-ip-v4" bootstrap_nic_ip_v6_configuration_name = "bootstrap-nic-ip-v6" description = "Created By OpenShift Installer" + tags = merge( + { + "kubernetes.io_cluster.${var.cluster_id}" = "owned" + }, + var.azure_extra_tags, + ) +} + +provider "azurerm" { + features {} + subscription_id = var.azure_subscription_id + client_id = var.azure_client_id + client_secret = var.azure_client_secret + tenant_id = var.azure_tenant_id + environment = var.azure_environment } data "azurerm_storage_account_sas" "ignition" { @@ -43,7 +58,7 @@ resource "azurerm_storage_container" "ignition" { } resource "local_file" "ignition_bootstrap" { - content = var.ignition + content = var.ignition_bootstrap filename = "${path.module}/ignition_bootstrap.ign" } @@ -62,27 +77,27 @@ data "ignition_config" "redirect" { } resource "azurerm_public_ip" "bootstrap_public_ip_v4" { - count = var.private || ! var.use_ipv4 ? 0 : 1 + count = var.azure_private || ! var.use_ipv4 ? 0 : 1 sku = "Standard" - location = var.region + location = var.azure_region name = "${var.cluster_id}-bootstrap-pip-v4" resource_group_name = var.resource_group_name allocation_method = "Static" } data "azurerm_public_ip" "bootstrap_public_ip_v4" { - count = var.private ? 0 : 1 + count = var.azure_private ? 0 : 1 name = azurerm_public_ip.bootstrap_public_ip_v4[0].name resource_group_name = var.resource_group_name } resource "azurerm_public_ip" "bootstrap_public_ip_v6" { - count = var.private || ! var.use_ipv6 ? 0 : 1 + count = var.azure_private || ! var.use_ipv6 ? 0 : 1 sku = "Standard" - location = var.region + location = var.azure_region name = "${var.cluster_id}-bootstrap-pip-v6" resource_group_name = var.resource_group_name allocation_method = "Static" @@ -90,7 +105,7 @@ resource "azurerm_public_ip" "bootstrap_public_ip_v6" { } data "azurerm_public_ip" "bootstrap_public_ip_v6" { - count = var.private || ! var.use_ipv6 ? 0 : 1 + count = var.azure_private || ! var.use_ipv6 ? 0 : 1 name = azurerm_public_ip.bootstrap_public_ip_v6[0].name resource_group_name = var.resource_group_name @@ -98,7 +113,7 @@ data "azurerm_public_ip" "bootstrap_public_ip_v6" { resource "azurerm_network_interface" "bootstrap" { name = "${var.cluster_id}-bootstrap-nic" - location = var.region + location = var.azure_region resource_group_name = var.resource_group_name dynamic "ip_configuration" { @@ -108,14 +123,14 @@ resource "azurerm_network_interface" "bootstrap" { primary : var.use_ipv4, name : local.bootstrap_nic_ip_v4_configuration_name, ip_address_version : "IPv4", - public_ip_id : var.private ? null : azurerm_public_ip.bootstrap_public_ip_v4[0].id, + public_ip_id : var.azure_private ? null : azurerm_public_ip.bootstrap_public_ip_v4[0].id, include : var.use_ipv4 || var.use_ipv6, }, { primary : ! var.use_ipv4, name : local.bootstrap_nic_ip_v6_configuration_name, ip_address_version : "IPv6", - public_ip_id : var.private || ! var.use_ipv6 ? null : azurerm_public_ip.bootstrap_public_ip_v6[0].id, + public_ip_id : var.azure_private || ! var.use_ipv6 ? null : azurerm_public_ip.bootstrap_public_ip_v6[0].id, include : var.use_ipv6, }, ] : { @@ -129,7 +144,7 @@ resource "azurerm_network_interface" "bootstrap" { content { primary = ip_configuration.value.primary name = ip_configuration.value.name - subnet_id = var.subnet_id + subnet_id = var.master_subnet_id private_ip_address_version = ip_configuration.value.ip_address_version private_ip_address_allocation = "Dynamic" public_ip_address_id = ip_configuration.value.public_ip_id @@ -140,7 +155,7 @@ resource "azurerm_network_interface" "bootstrap" { resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v4" { // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf` // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570 - count = (! var.private || ! var.outbound_udr) ? 1 : 0 + count = (! var.azure_private || ! var.azure_outbound_user_defined_routing) ? 1 : 0 network_interface_id = azurerm_network_interface.bootstrap.id backend_address_pool_id = var.elb_backend_pool_v4_id @@ -150,7 +165,7 @@ resource "azurerm_network_interface_backend_address_pool_association" "public_lb resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v6" { // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf` // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570 - count = var.use_ipv6 && (! var.private || ! var.outbound_udr) ? 1 : 0 + count = var.use_ipv6 && (! var.azure_private || ! var.azure_outbound_user_defined_routing) ? 1 : 0 network_interface_id = azurerm_network_interface.bootstrap.id backend_address_pool_id = var.elb_backend_pool_v6_id @@ -175,10 +190,10 @@ resource "azurerm_network_interface_backend_address_pool_association" "internal_ resource "azurerm_linux_virtual_machine" "bootstrap" { name = "${var.cluster_id}-bootstrap" - location = var.region + location = var.azure_region resource_group_name = var.resource_group_name network_interface_ids = [azurerm_network_interface.bootstrap.id] - size = var.vm_size + size = var.azure_bootstrap_vm_type admin_username = "core" # The password is normally applied by WALA (the Azure agent), but this # isn't installed in RHCOS. As a result, this password is never set. It is @@ -216,7 +231,7 @@ resource "azurerm_linux_virtual_machine" "bootstrap" { } resource "azurerm_network_security_rule" "bootstrap_ssh_in" { - count = var.private ? 0 : 1 + count = var.azure_private ? 0 : 1 name = "bootstrap_ssh_in" priority = 103 diff --git a/data/data/azure/bootstrap/outputs.tf b/data/data/azure/bootstrap/outputs.tf new file mode 100644 index 00000000000..92c0add6b64 --- /dev/null +++ b/data/data/azure/bootstrap/outputs.tf @@ -0,0 +1,3 @@ +output "bootstrap_ip" { + value = var.azure_private ? azurerm_network_interface.bootstrap.private_ip_address : azurerm_public_ip.bootstrap_public_ip_v4[0].ip_address +} diff --git a/data/data/azure/bootstrap/variables.tf b/data/data/azure/bootstrap/variables.tf index fbe0a9b2908..c319f987d45 100644 --- a/data/data/azure/bootstrap/variables.tf +++ b/data/data/azure/bootstrap/variables.tf @@ -1,72 +1,30 @@ -variable "vm_size" { - type = string - description = "The SKU ID for the bootstrap node." -} - -variable "vm_image" { - type = string - description = "The resource id of the vm image used for bootstrap." -} - -variable "region" { - type = string - description = "The region for the deployment." -} - -variable "resource_group_name" { - type = string - description = "The resource group name for the deployment." -} - -variable "cluster_id" { - type = string - description = "The identifier for the cluster." -} - -variable "identity" { - type = string - description = "The user assigned identity id for the vm." -} - -variable "ignition" { - type = string - description = "The content of the bootstrap ignition file." -} - -variable "subnet_id" { - type = string - description = "The subnet ID for the bootstrap node." -} - variable "elb_backend_pool_v4_id" { type = string + default = null description = "The external load balancer bakend pool id. used to attach the bootstrap NIC" } variable "elb_backend_pool_v6_id" { type = string + default = null description = "The external load balancer bakend pool id for ipv6. used to attach the bootstrap NIC" } variable "ilb_backend_pool_v4_id" { type = string + default = null description = "The internal load balancer bakend pool id. used to attach the bootstrap NIC" } variable "ilb_backend_pool_v6_id" { type = string + default = null description = "The internal load balancer bakend pool id for ipv6. used to attach the bootstrap NIC" } -variable "storage_account" { - type = any - description = "the storage account for the cluster. It can be used for boot diagnostics." -} - -variable "tags" { - type = map(string) - default = {} - description = "tags to be applied to created resources." +variable "master_subnet_id" { + type = string + description = "The subnet ID for the bootstrap node." } variable "nsg_name" { @@ -74,19 +32,24 @@ variable "nsg_name" { description = "The network security group for the subnet." } -variable "private" { - type = bool - description = "This value determines if this is a private cluster or not." +variable "resource_group_name" { + type = string + description = "The resource group name for the deployment." +} + +variable "storage_account" { + type = any + description = "the storage account for the cluster. It can be used for boot diagnostics." } -variable "use_ipv4" { - type = bool - description = "This value determines if this is cluster should use IPv4 networking." +variable "vm_image" { + type = string + description = "The resource id of the vm image used for bootstrap." } -variable "use_ipv6" { - type = bool - description = "This value determines if this is cluster should use IPv6 networking." +variable "identity" { + type = string + description = "The user assigned identity id for the vm." } variable "outbound_udr" { @@ -94,10 +57,14 @@ variable "outbound_udr" { default = false description = <