Azure Stack IPI Support

data/data/azure/variables-azure.tf

@@ -61,6 +61,12 @@ variable "azure_image_url" {
   description = "The URL of the vm image used for all nodes."
 }
 
+variable "azure_arm_endpoint" {
+  type        = string
+  default     = ""
+  description = "The endpoint for the Azure API. Only used when installing to Azure Stack"
+}
+
 variable "azure_subscription_id" {
   type        = string
   description = "The subscription that should be used to interact with Azure API"

data/data/azurestack/bootstrap/main.tf

@@ -0,0 +1,171 @@
+locals {
+  bootstrap_nic_ip_v4_configuration_name = "bootstrap-nic-ip-v4"
+  description                            = "Created By OpenShift Installer"
+  tags = merge(
+    {
+      "kubernetes.io_cluster.${var.cluster_id}" = "owned"
+    },
+    var.azure_extra_tags,
+  )
+}
+
+provider "azurestack" {
+  arm_endpoint    = var.azure_arm_endpoint
+  subscription_id = var.azure_subscription_id
+  client_id       = var.azure_client_id
+  client_secret   = var.azure_client_secret
+  tenant_id       = var.azure_tenant_id
+}
+
+data "azurestack_storage_account_sas" "ignition" {
+  connection_string = var.storage_account.primary_connection_string
+  https_only        = true
+
+  resource_types {
+    service   = false
+    container = false
+    object    = true
+  }
+
+  services {
+    blob  = true
+    queue = false
+    table = false
+    file  = false
+  }
+
+  start  = timestamp()
+  expiry = timeadd(timestamp(), "24h")
+
+  permissions {
+    read    = true
+    list    = true
+    create  = false
+    add     = false
+    delete  = false
+    process = false
+    write   = false
+    update  = false
+  }
+}
+
+resource "azurestack_storage_container" "ignition" {
+  name                  = "ignition"
+  resource_group_name   = var.resource_group_name
+  storage_account_name  = var.storage_account.name
+  container_access_type = "private"
+}
+
+resource "local_file" "ignition_bootstrap" {
+  content  = var.ignition_bootstrap
+  filename = "${path.module}/ignition_bootstrap.ign"
+}
+
+resource "azurestack_storage_blob" "ignition" {
+  name                   = "bootstrap.ign"
+  source                 = local_file.ignition_bootstrap.filename
+  resource_group_name    = var.resource_group_name
+  storage_account_name   = var.storage_account.name
+  storage_container_name = azurestack_storage_container.ignition.name
+  type                   = "block"
+}
+
+data "ignition_config" "redirect" {
+  replace {
+    source = "${azurestack_storage_blob.ignition.url}${data.azurestack_storage_account_sas.ignition.sas}"
+  }
+}
+
+resource "azurestack_public_ip" "bootstrap_public_ip_v4" {
+  count = var.azure_private ? 0 : 1
+
+  location                     = var.azure_region
+  name                         = "${var.cluster_id}-bootstrap-pip-v4"
+  resource_group_name          = var.resource_group_name
+  public_ip_address_allocation = "Static"
+}
+
+data "azurestack_public_ip" "bootstrap_public_ip_v4" {
+  count = var.azure_private ? 0 : 1
+
+  name                = azurestack_public_ip.bootstrap_public_ip_v4[0].name
+  resource_group_name = var.resource_group_name
+}
+
+resource "azurestack_network_interface" "bootstrap" {
+  name                = "${var.cluster_id}-bootstrap-nic"
+  location            = var.azure_region
+  resource_group_name = var.resource_group_name
+
+  ip_configuration {
+    primary                       = true
+    name                          = local.bootstrap_nic_ip_v4_configuration_name
+    subnet_id                     = var.master_subnet_id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id          = var.azure_private ? null : azurestack_public_ip.bootstrap_public_ip_v4[0].id
+    load_balancer_backend_address_pools_ids = concat(
+      [var.ilb_backend_pool_v4_id],
+      ! var.azure_private ? [var.elb_backend_pool_v4_id] : null
+    )
+  }
+}
+
+resource "azurestack_virtual_machine" "bootstrap" {
+  name                  = "${var.cluster_id}-bootstrap"
+  location              = var.azure_region
+  resource_group_name   = var.resource_group_name
+  network_interface_ids = [azurestack_network_interface.bootstrap.id]
+  vm_size               = var.azure_bootstrap_vm_type
+  availability_set_id   = var.availability_set_id
+
+  os_profile {
+    computer_name  = "${var.cluster_id}-bootstrap-vm"
+    admin_username = "core"
+    # The password is normally applied by WALA (the Azure agent), but this
+    # isn't installed in RHCOS. As a result, this password is never set. It is
+    # included here because it is required by the Azure ARM API.
+    admin_password = "NotActuallyApplied!"
+    custom_data    = base64encode(data.ignition_config.redirect.rendered)
+  }
+
+  os_profile_linux_config {
+    disable_password_authentication = false
+  }
+
+  storage_image_reference {
+    id = var.vm_image
+  }
+
+  storage_os_disk {
+    name              = "${var.cluster_id}-bootstrap_OSDisk" # os disk name needs to match cluster-api convention
+    create_option     = "FromImage"
+    disk_size_gb      = 100
+    managed_disk_type = "Standard_LRS"
+  }
+
+  boot_diagnostics {
+    enabled     = true
+    storage_uri = var.storage_account.primary_blob_endpoint
+  }
+
+  # Workaround for bug in provider where destroy fails by trying to delete NIC before VM.
+  # This depends_on ensures the VM is destroyed before the NIC.
+  depends_on = [
+    azurestack_network_interface.bootstrap
+  ]
+}
+
+resource "azurestack_network_security_rule" "bootstrap_ssh_in" {
+  name                        = "bootstrap_ssh_in"
+  priority                    = 103
+  direction                   = "Inbound"
+  access                      = "Allow"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "22"
+  source_address_prefix       = "*"
+  destination_address_prefix  = "*"
+  resource_group_name         = var.resource_group_name
+  network_security_group_name = var.nsg_name
+  description                 = local.description
+}

data/data/azurestack/bootstrap/outputs.tf

@@ -0,0 +1,3 @@
+output "bootstrap_ip" {
+  value = var.azure_private ? azurestack_network_interface.bootstrap.private_ip_address : azurestack_public_ip.bootstrap_public_ip_v4[0].ip_address
+}

data/data/azurestack/bootstrap/variables.tf

@@ -0,0 +1,39 @@
+variable "elb_backend_pool_v4_id" {
+  type        = string
+  description = "The external load balancer backend pool id. used to attach the bootstrap NIC"
+}
+
+variable "ilb_backend_pool_v4_id" {
+  type        = string
+  description = "The internal load balancer backend pool id. used to attach the bootstrap NIC"
+}
+
+variable "master_subnet_id" {
+  type        = string
+  description = "The subnet ID for the bootstrap node."
+}
+
+variable "nsg_name" {
+  type        = string
+  description = "The network security group for the subnet."
+}
+
+variable "resource_group_name" {
+  type        = string
+  description = "The resource group name for the deployment."
+}
+
+variable "storage_account" {
+  type        = any
+  description = "the storage account for the cluster. It can be used for boot diagnostics."
+}
+
+variable "vm_image" {
+  type        = string
+  description = "The URI of the vm image to used for bootstrap."
+}
+
+variable "availability_set_id" {
+  type        = string
+  description = "ID of the availability set in which to place VMs"
+}

data/data/azurestack/bootstrap/versions.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_version = ">= 0.12"
+  required_providers {
+    local = {
+      source = "openshift/local/local"
+    }
+    azurestack = {
+      source = "openshift/local/azurestack"
+    }
+    ignition = {
+      source = "openshift/local/ignition"
+    }
+  }
+}
+

data/data/azurestack/cluster/dns/dns.tf

@@ -0,0 +1,24 @@
+locals {
+  // extracting <clustername> from <clusterdomain>
+  cluster_name = replace(var.cluster_domain, ".${var.base_domain}", "")
+}
+
+resource "azurestack_dns_a_record" "api_external_v4" {
+  count = ! var.private ? 1 : 0
+
+  name                = "api.${local.cluster_name}"
+  zone_name           = var.base_domain
+  resource_group_name = var.base_domain_resource_group_name
+  ttl                 = 300
+  records             = var.private ? [var.ilb_ipaddress_v4] : [var.elb_pip_v4]
+  tags                = var.tags
+}
+
+resource "azurestack_dns_a_record" "api_internal_v4" {
+  name                = "api-int.${local.cluster_name}"
+  zone_name           = var.base_domain
+  resource_group_name = var.base_domain_resource_group_name
+  ttl                 = 300
+  records             = [var.ilb_ipaddress_v4]
+  tags                = var.tags
+}

data/data/azurestack/cluster/dns/variables.tf

@@ -0,0 +1,55 @@
+variable "tags" {
+  type        = map(string)
+  default     = {}
+  description = "tags to be applied to created resources."
+}
+
+variable "cluster_id" {
+  description = "The identifier for the cluster."
+  type        = string
+}
+
+variable "cluster_domain" {
+  description = "The domain for the cluster that all DNS records must belong"
+  type        = string
+}
+
+variable "base_domain" {
+  description = "The base domain used for public records"
+  type        = string
+}
+
+variable "base_domain_resource_group_name" {
+  description = "The resource group where the base domain is"
+  type        = string
+}
+
+variable "elb_fqdn_v4" {
+  description = "External API's LB fqdn for IPv4"
+  type        = string
+}
+
+variable "elb_pip_v4" {
+  description = "Public IP address of the external API's LB"
+  type        = string
+}
+
+variable "ilb_ipaddress_v4" {
+  description = "Internal API's LB IP v4 address"
+  type        = string
+}
+
+variable "virtual_network_id" {
+  description = "The ID for Virtual Network that will be linked to the Private DNS zone."
+  type        = string
+}
+
+variable "resource_group_name" {
+  type        = string
+  description = "Resource group for the deployment"
+}
+
+variable "private" {
+  type        = bool
+  description = "This value determines if this is a private cluster or not."
+}

data/data/azurestack/cluster/dns/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 0.12"
+  required_providers {
+    azurestack = {
+      source = "openshift/local/azurestack"
+    }
+  }
+}
+

data/data/azurestack/cluster/main.tf

@@ -0,0 +1,51 @@
+locals {
+  tags = merge(
+    {
+      "kubernetes.io_cluster.${var.cluster_id}" = "owned"
+    },
+    var.azure_extra_tags,
+  )
+  description = "Created By OpenShift Installer"
+}
+
+provider "azurestack" {
+  arm_endpoint    = var.azure_arm_endpoint
+  subscription_id = var.azure_subscription_id
+  client_id       = var.azure_client_id
+  client_secret   = var.azure_client_secret
+  tenant_id       = var.azure_tenant_id
+}
+
+
+module "master" {
+  source                 = "./master"
+  resource_group_name    = var.resource_group_name
+  cluster_id             = var.cluster_id
+  region                 = var.azure_region
+  vm_size                = var.azure_master_vm_type
+  vm_image_uri           = var.vm_image
+  ignition               = var.ignition_master
+  elb_backend_pool_v4_id = var.elb_backend_pool_v4_id
+  ilb_backend_pool_v4_id = var.ilb_backend_pool_v4_id
+  subnet_id              = var.master_subnet_id
+  instance_count         = var.master_count
+  storage_account        = var.storage_account
+  os_volume_size         = var.azure_master_root_volume_size
+  private                = var.azure_private
+  availability_set_id    = var.availability_set_id
+}
+
+module "dns" {
+  source                          = "./dns"
+  cluster_domain                  = var.cluster_domain
+  cluster_id                      = var.cluster_id
+  base_domain                     = var.base_domain
+  virtual_network_id              = var.virtual_network_id
+  elb_fqdn_v4                     = var.elb_pip_v4_fqdn
+  elb_pip_v4                      = var.elb_pip_v4
+  ilb_ipaddress_v4                = var.ilb_ip_v4_address
+  resource_group_name             = var.resource_group_name
+  base_domain_resource_group_name = var.azure_base_domain_resource_group_name
+  private                         = var.azure_private
+  tags                            = local.tags
+}