Azure: Split terraform into stages

data/data/azure/bootstrap/main.tf

@@ -2,6 +2,21 @@ locals {
   bootstrap_nic_ip_v4_configuration_name = "bootstrap-nic-ip-v4"
   bootstrap_nic_ip_v6_configuration_name = "bootstrap-nic-ip-v6"
   description                            = "Created By OpenShift Installer"
+  tags = merge(
+    {
+      "kubernetes.io_cluster.${var.cluster_id}" = "owned"
+    },
+    var.azure_extra_tags,
+  )
+}
+
+provider "azurerm" {
+  features {}
+  subscription_id = var.azure_subscription_id
+  client_id       = var.azure_client_id
+  client_secret   = var.azure_client_secret
+  tenant_id       = var.azure_tenant_id
+  environment     = var.azure_environment
 }
 
 data "azurerm_storage_account_sas" "ignition" {
@@ -43,7 +58,7 @@ resource "azurerm_storage_container" "ignition" {
 }
 
 resource "local_file" "ignition_bootstrap" {
-  content  = var.ignition
+  content  = var.ignition_bootstrap
   filename = "${path.module}/ignition_bootstrap.ign"
 }
 
@@ -62,43 +77,43 @@ data "ignition_config" "redirect" {
 }
 
 resource "azurerm_public_ip" "bootstrap_public_ip_v4" {
-  count = var.private || ! var.use_ipv4 ? 0 : 1
+  count = var.azure_private || ! var.use_ipv4 ? 0 : 1
 
   sku                 = "Standard"
-  location            = var.region
+  location            = var.azure_region
   name                = "${var.cluster_id}-bootstrap-pip-v4"
   resource_group_name = var.resource_group_name
   allocation_method   = "Static"
 }
 
 data "azurerm_public_ip" "bootstrap_public_ip_v4" {
-  count = var.private ? 0 : 1
+  count = var.azure_private ? 0 : 1
 
   name                = azurerm_public_ip.bootstrap_public_ip_v4[0].name
   resource_group_name = var.resource_group_name
 }
 
 resource "azurerm_public_ip" "bootstrap_public_ip_v6" {
-  count = var.private || ! var.use_ipv6 ? 0 : 1
+  count = var.azure_private || ! var.use_ipv6 ? 0 : 1
 
   sku                 = "Standard"
-  location            = var.region
+  location            = var.azure_region
   name                = "${var.cluster_id}-bootstrap-pip-v6"
   resource_group_name = var.resource_group_name
   allocation_method   = "Static"
   ip_version          = "IPv6"
 }
 
 data "azurerm_public_ip" "bootstrap_public_ip_v6" {
-  count = var.private || ! var.use_ipv6 ? 0 : 1
+  count = var.azure_private || ! var.use_ipv6 ? 0 : 1
 
   name                = azurerm_public_ip.bootstrap_public_ip_v6[0].name
   resource_group_name = var.resource_group_name
 }
 
 resource "azurerm_network_interface" "bootstrap" {
   name                = "${var.cluster_id}-bootstrap-nic"
-  location            = var.region
+  location            = var.azure_region
   resource_group_name = var.resource_group_name
 
   dynamic "ip_configuration" {
@@ -108,14 +123,14 @@ resource "azurerm_network_interface" "bootstrap" {
         primary : var.use_ipv4,
         name : local.bootstrap_nic_ip_v4_configuration_name,
         ip_address_version : "IPv4",
-        public_ip_id : var.private ? null : azurerm_public_ip.bootstrap_public_ip_v4[0].id,
+        public_ip_id : var.azure_private ? null : azurerm_public_ip.bootstrap_public_ip_v4[0].id,
         include : var.use_ipv4 || var.use_ipv6,
       },
       {
         primary : ! var.use_ipv4,
         name : local.bootstrap_nic_ip_v6_configuration_name,
         ip_address_version : "IPv6",
-        public_ip_id : var.private || ! var.use_ipv6 ? null : azurerm_public_ip.bootstrap_public_ip_v6[0].id,
+        public_ip_id : var.azure_private || ! var.use_ipv6 ? null : azurerm_public_ip.bootstrap_public_ip_v6[0].id,
         include : var.use_ipv6,
       },
       ] : {
@@ -129,7 +144,7 @@ resource "azurerm_network_interface" "bootstrap" {
     content {
       primary                       = ip_configuration.value.primary
       name                          = ip_configuration.value.name
-      subnet_id                     = var.subnet_id
+      subnet_id                     = var.master_subnet_id
       private_ip_address_version    = ip_configuration.value.ip_address_version
       private_ip_address_allocation = "Dynamic"
       public_ip_address_id          = ip_configuration.value.public_ip_id
@@ -140,7 +155,7 @@ resource "azurerm_network_interface" "bootstrap" {
 resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v4" {
   // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
   // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
-  count = (! var.private || ! var.outbound_udr) ? 1 : 0
+  count = (! var.azure_private || ! var.azure_outbound_user_defined_routing) ? 1 : 0
 
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.elb_backend_pool_v4_id
@@ -150,7 +165,7 @@ resource "azurerm_network_interface_backend_address_pool_association" "public_lb
 resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v6" {
   // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
   // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
-  count = var.use_ipv6 && (! var.private || ! var.outbound_udr) ? 1 : 0
+  count = var.use_ipv6 && (! var.azure_private || ! var.azure_outbound_user_defined_routing) ? 1 : 0
 
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.elb_backend_pool_v6_id
@@ -175,10 +190,10 @@ resource "azurerm_network_interface_backend_address_pool_association" "internal_
 
 resource "azurerm_linux_virtual_machine" "bootstrap" {
   name                  = "${var.cluster_id}-bootstrap"
-  location              = var.region
+  location              = var.azure_region
   resource_group_name   = var.resource_group_name
   network_interface_ids = [azurerm_network_interface.bootstrap.id]
-  size                  = var.vm_size
+  size                  = var.azure_bootstrap_vm_type
   admin_username        = "core"
   # The password is normally applied by WALA (the Azure agent), but this
   # isn't installed in RHCOS. As a result, this password is never set. It is
@@ -216,7 +231,7 @@ resource "azurerm_linux_virtual_machine" "bootstrap" {
 }
 
 resource "azurerm_network_security_rule" "bootstrap_ssh_in" {
-  count = var.private ? 0 : 1
+  count = var.azure_private ? 0 : 1
 
   name                        = "bootstrap_ssh_in"
   priority                    = 103

data/data/azure/bootstrap/outputs.tf

@@ -0,0 +1,3 @@
+output "bootstrap_ip" {
+  value = var.azure_private ? azurerm_network_interface.bootstrap.private_ip_address : azurerm_public_ip.bootstrap_public_ip_v4[0].ip_address
+}

data/data/azure/bootstrap/variables.tf

@@ -1,103 +1,70 @@
-variable "vm_size" {
-  type        = string
-  description = "The SKU ID for the bootstrap node."
-}
-
-variable "vm_image" {
-  type        = string
-  description = "The resource id of the vm image used for bootstrap."
-}
-
-variable "region" {
-  type        = string
-  description = "The region for the deployment."
-}
-
-variable "resource_group_name" {
-  type        = string
-  description = "The resource group name for the deployment."
-}
-
-variable "cluster_id" {
-  type        = string
-  description = "The identifier for the cluster."
-}
-
-variable "identity" {
-  type        = string
-  description = "The user assigned identity id for the vm."
-}
-
-variable "ignition" {
-  type        = string
-  description = "The content of the bootstrap ignition file."
-}
-
-variable "subnet_id" {
-  type        = string
-  description = "The subnet ID for the bootstrap node."
-}
-
 variable "elb_backend_pool_v4_id" {
   type        = string
+  default     = null
   description = "The external load balancer bakend pool id. used to attach the bootstrap NIC"
 }
 
 variable "elb_backend_pool_v6_id" {
   type        = string
+  default     = null
   description = "The external load balancer bakend pool id for ipv6. used to attach the bootstrap NIC"
 }
 
 variable "ilb_backend_pool_v4_id" {
   type        = string
+  default     = null
   description = "The internal load balancer bakend pool id. used to attach the bootstrap NIC"
 }
 
 variable "ilb_backend_pool_v6_id" {
   type        = string
+  default     = null
   description = "The internal load balancer bakend pool id for ipv6. used to attach the bootstrap NIC"
 }
 
-variable "storage_account" {
-  type        = any
-  description = "the storage account for the cluster. It can be used for boot diagnostics."
-}
-
-variable "tags" {
-  type        = map(string)
-  default     = {}
-  description = "tags to be applied to created resources."
+variable "master_subnet_id" {
+  type        = string
+  description = "The subnet ID for the bootstrap node."
 }
 
 variable "nsg_name" {
   type        = string
   description = "The network security group for the subnet."
 }
 
-variable "private" {
-  type        = bool
-  description = "This value determines if this is a private cluster or not."
+variable "resource_group_name" {
+  type        = string
+  description = "The resource group name for the deployment."
+}
+
+variable "storage_account" {
+  type        = any
+  description = "the storage account for the cluster. It can be used for boot diagnostics."
 }
 
-variable "use_ipv4" {
-  type        = bool
-  description = "This value determines if this is cluster should use IPv4 networking."
+variable "vm_image" {
+  type        = string
+  description = "The resource id of the vm image used for bootstrap."
 }
 
-variable "use_ipv6" {
-  type        = bool
-  description = "This value determines if this is cluster should use IPv6 networking."
+variable "identity" {
+  type        = string
+  description = "The user assigned identity id for the vm."
 }
 
 variable "outbound_udr" {
   type    = bool
   default = false
 
   description = <<EOF
-This determined whether User defined routing will be used for egress to Internet.
+This determined whether User defined routing will be used for egress to
+Internet.
 When false, Standard LB will be used for egress to the Internet.
 
-This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
-conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+This is required because terraform cannot calculate counts during plan phase
+completely and therefore the `vnet/public-lb.tf`
+conditional need to be recreated. See
+https://github.com/hashicorp/terraform/issues/12570
 EOF
 }
+

data/data/azure/cluster/main.tf

@@ -0,0 +1,73 @@
+locals {
+  tags = merge(
+    {
+      "kubernetes.io_cluster.${var.cluster_id}" = "owned"
+    },
+    var.azure_extra_tags,
+  )
+  description = "Created By OpenShift Installer"
+  # At this time min_tls_version is only supported in the Public Cloud and US Government Cloud.
+  environments_with_min_tls_version = ["public", "usgovernment"]
+
+}
+
+provider "azurerm" {
+  features {}
+  subscription_id = var.azure_subscription_id
+  client_id       = var.azure_client_id
+  client_secret   = var.azure_client_secret
+  tenant_id       = var.azure_tenant_id
+  environment     = var.azure_environment
+}
+
+provider "azureprivatedns" {
+  subscription_id = var.azure_subscription_id
+  client_id       = var.azure_client_id
+  client_secret   = var.azure_client_secret
+  tenant_id       = var.azure_tenant_id
+  environment     = var.azure_environment
+}
+
+module "master" {
+  source                 = "./master"
+  resource_group_name    = var.resource_group_name
+  cluster_id             = var.cluster_id
+  region                 = var.azure_region
+  availability_zones     = var.azure_master_availability_zones
+  vm_size                = var.azure_master_vm_type
+  vm_image               = var.vm_image
+  identity               = var.identity
+  ignition               = var.ignition_master
+  elb_backend_pool_v4_id = var.elb_backend_pool_v4_id
+  elb_backend_pool_v6_id = var.elb_backend_pool_v6_id
+  ilb_backend_pool_v4_id = var.ilb_backend_pool_v4_id
+  ilb_backend_pool_v6_id = var.ilb_backend_pool_v6_id
+  subnet_id              = var.master_subnet_id
+  instance_count         = var.master_count
+  storage_account        = var.storage_account
+  os_volume_type         = var.azure_master_root_volume_type
+  os_volume_size         = var.azure_master_root_volume_size
+  private                = var.azure_private
+  outbound_udr           = var.azure_outbound_user_defined_routing
+
+  use_ipv4 = var.use_ipv4
+  use_ipv6 = var.use_ipv6
+}
+
+module "dns" {
+  source                          = "./dns"
+  cluster_domain                  = var.cluster_domain
+  cluster_id                      = var.cluster_id
+  base_domain                     = var.base_domain
+  virtual_network_id              = var.virtual_network_id
+  external_lb_fqdn_v4             = var.public_lb_pip_v4_fqdn
+  external_lb_fqdn_v6             = var.public_lb_pip_v6_fqdn
+  internal_lb_ipaddress_v4        = var.internal_lb_ip_v4_address
+  internal_lb_ipaddress_v6        = var.internal_lb_ip_v6_address
+  resource_group_name             = var.resource_group_name
+  base_domain_resource_group_name = var.azure_base_domain_resource_group_name
+  private                         = var.azure_private
+
+  use_ipv4 = var.use_ipv4
+  use_ipv6 = var.use_ipv6
+}

data/data/azure/cluster/master/outputs.tf

@@ -0,0 +1,3 @@
+output "ip_addresses" {
+  value = azurerm_network_interface.master.*.private_ip_addresses
+}

data/data/azure/cluster/outputs.tf

@@ -0,0 +1,3 @@
+output "control_plane_ips" {
+  value = module.master.ip_addresses
+}