Azure: Create clusters with no public endpoints

data/data/azure/bootstrap/main.tf

@@ -62,6 +62,8 @@ data "ignition_config" "redirect" {
 }
 
 resource "azurerm_public_ip" "bootstrap_public_ip" {
+  count = var.private ? 0 : 1
+
   sku                 = "Standard"
   location            = var.region
   name                = "${var.cluster_id}-bootstrap-pip"
@@ -70,7 +72,9 @@ resource "azurerm_public_ip" "bootstrap_public_ip" {
 }
 
 data "azurerm_public_ip" "bootstrap_public_ip" {
-  name                = azurerm_public_ip.bootstrap_public_ip.name
+  count = var.private ? 0 : 1
+
+  name                = azurerm_public_ip.bootstrap_public_ip[0].name
   resource_group_name = var.resource_group_name
 }
 
@@ -83,7 +87,7 @@ resource "azurerm_network_interface" "bootstrap" {
     subnet_id                     = var.subnet_id
     name                          = local.bootstrap_nic_ip_configuration_name
     private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = azurerm_public_ip.bootstrap_public_ip.id
+    public_ip_address_id          = var.private ? null : azurerm_public_ip.bootstrap_public_ip[0].id
   }
 }
 

data/data/azure/bootstrap/variables.tf

@@ -63,3 +63,8 @@ variable "nsg_name" {
   type        = string
   description = "The network security group for the subnet."
 }
+
+variable "private" {
+  type        = bool
+  description = "This value determines if this is a private cluster or not."
+}

data/data/azure/dns/dns.tf

@@ -32,6 +32,8 @@ resource "azureprivatedns_a_record" "api_internal" {
 }
 
 resource "azurerm_dns_cname_record" "api_external" {
+  count = var.private ? 0 : 1
+
   name                = local.api_external_name
   zone_name           = var.base_domain
   resource_group_name = var.base_domain_resource_group_name

data/data/azure/dns/variables.tf

@@ -54,3 +54,8 @@ variable "resource_group_name" {
   type        = string
   description = "Resource group for the deployment"
 }
+
+variable "private" {
+  type        = bool
+  description = "This value determines if this is a private cluster or not."
+}

data/data/azure/main.tf

@@ -36,6 +36,7 @@ module "bootstrap" {
   tags                = local.tags
   storage_account     = azurerm_storage_account.cluster
   nsg_name            = module.vnet.master_nsg_name
+  private             = module.vnet.private
 }
 
 module "vnet" {
@@ -51,6 +52,7 @@ module "vnet" {
   virtual_network_name        = var.azure_virtual_network
   master_subnet               = var.azure_control_plane_subnet
   worker_subnet               = var.azure_compute_subnet
+  private                     = var.azure_private
 }
 
 module "master" {
@@ -71,6 +73,7 @@ module "master" {
   storage_account     = azurerm_storage_account.cluster
   os_volume_type      = var.azure_master_root_volume_type
   os_volume_size      = var.azure_master_root_volume_size
+  private             = module.vnet.private
 }
 
 module "dns" {
@@ -85,6 +88,7 @@ module "dns" {
   base_domain_resource_group_name = var.azure_base_domain_resource_group_name
   etcd_count                      = var.master_count
   etcd_ip_addresses               = module.master.ip_addresses
+  private                         = module.vnet.private
 }
 
 resource "random_string" "storage_suffix" {

data/data/azure/master/master.tf

@@ -19,21 +19,24 @@ resource "azurerm_network_interface" "master" {
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "master" {
-  count                   = var.instance_count
+  count = var.instance_count
+
   network_interface_id    = element(azurerm_network_interface.master.*.id, count.index)
   backend_address_pool_id = var.elb_backend_pool_id
   ip_configuration_name   = local.ip_configuration_name #must be the same as nic's ip configuration name.
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "master_internal" {
-  count                   = var.instance_count
+  count = var.instance_count
+
   network_interface_id    = element(azurerm_network_interface.master.*.id, count.index)
   backend_address_pool_id = var.ilb_backend_pool_id
   ip_configuration_name   = local.ip_configuration_name #must be the same as nic's ip configuration name.
 }
 
 resource "azurerm_virtual_machine" "master" {
-  count                 = var.instance_count
+  count = var.instance_count
+
   name                  = "${var.cluster_id}-master-${count.index}"
   location              = var.region
   zones                 = compact([var.availability_zones[count.index]])

data/data/azure/master/variables.tf

@@ -86,3 +86,8 @@ variable "availability_zones" {
   type        = list(string)
   description = "List of the availability zones in which to create the masters. The length of this list must match instance_count."
 }
+
+variable "private" {
+  type        = bool
+  description = "This value determines if this is a private cluster or not."
+}

data/data/azure/variables-azure.tf

@@ -106,3 +106,8 @@ variable "azure_compute_subnet" {
   type        = string
   description = "The name of the subnet for worker nodes, either existing or to be created"
 }
+
+variable "azure_private" {
+  type        = bool
+  description = "This determines if this is a private cluster or not."
+}

data/data/azure/vnet/outputs.tf

@@ -1,5 +1,5 @@
 output "cluster-pip" {
-  value = azurerm_public_ip.cluster_public_ip.ip_address
+  value = var.private ? null : azurerm_public_ip.cluster_public_ip.ip_address
 }
 
 output "public_lb_backend_pool_id" {
@@ -11,11 +11,11 @@ output "internal_lb_backend_pool_id" {
 }
 
 output "public_lb_id" {
-  value = azurerm_lb.public.id
+  value = var.private ? null : azurerm_lb.public.id
 }
 
 output "public_lb_pip_fqdn" {
-  value = data.azurerm_public_ip.cluster_public_ip.fqdn
+  value = var.private ? null : data.azurerm_public_ip.cluster_public_ip.fqdn
 }
 
 output "internal_lb_ip_address" {
@@ -37,3 +37,7 @@ output "master_subnet_id" {
 output "worker_subnet_id" {
   value = local.worker_subnet_id
 }
+
+output "private" {
+  value = var.private
+}

data/data/azure/vnet/public-lb.tf

@@ -35,6 +35,8 @@ resource "azurerm_lb_backend_address_pool" "master_public_lb_pool" {
 }
 
 resource "azurerm_lb_rule" "public_lb_rule_api_internal" {
+  count = var.private ? 0 : 1
+
   name                           = "api-internal"
   resource_group_name            = var.resource_group_name
   protocol                       = "Tcp"
@@ -46,10 +48,28 @@ resource "azurerm_lb_rule" "public_lb_rule_api_internal" {
   enable_floating_ip             = false
   idle_timeout_in_minutes        = 30
   load_distribution              = "Default"
-  probe_id                       = azurerm_lb_probe.public_lb_probe_api_internal.id
+  probe_id                       = azurerm_lb_probe.public_lb_probe_api_internal[0].id
+}
+
+resource "azurerm_lb_rule" "internal_outbound_rule" {
+  count = var.private ? 1 : 0
+
+  name                           = "internal_outbound_rule"
+  resource_group_name            = var.resource_group_name
+  protocol                       = "Tcp"
+  backend_address_pool_id        = azurerm_lb_backend_address_pool.master_public_lb_pool.id
+  loadbalancer_id                = azurerm_lb.public.id
+  frontend_port                  = 27627
+  backend_port                   = 27627
+  frontend_ip_configuration_name = local.public_lb_frontend_ip_configuration_name
+  enable_floating_ip             = false
+  idle_timeout_in_minutes        = 30
+  load_distribution              = "Default"
 }
 
 resource "azurerm_lb_probe" "public_lb_probe_api_internal" {
+  count = var.private ? 0 : 1
+
   name                = "api-internal-probe"
   resource_group_name = var.resource_group_name
   interval_in_seconds = 10

data/data/azure/vnet/variables.tf

@@ -52,3 +52,8 @@ variable "worker_subnet" {
   type        = string
   description = "This is the name of the subnet used for the compute nodes, new or existing"
 }
+
+variable "private" {
+  type        = bool
+  description = "The determines if this is a private/internal cluster or not."
+}

data/data/manifests/openshift/private-cluster-outbound-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: openshift-config-managed
+  name: outbound-provider
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 27627

pkg/asset/cluster/tfvars.go

@@ -223,6 +223,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				WorkerConfigs:               workerConfigs,
 				ImageURL:                    string(*rhcosImage),
 				PreexistingNetwork:          preexistingnetwork,
+				Publish:                     installConfig.Config.Publish,
 			},
 		)
 		if err != nil {

pkg/asset/manifests/cloudproviderconfig.go

@@ -112,7 +112,6 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 		if installConfig.Config.Azure.ComputeSubnet != "" {
 			subnet = installConfig.Config.Azure.ComputeSubnet
 		}
-
 		azureConfig, err := azure.CloudProviderConfig{
 			GroupLocation:            installConfig.Config.Azure.Region,
 			ResourcePrefix:           clusterID.InfraID,

pkg/asset/manifests/dns.go

@@ -96,9 +96,11 @@ func (d *DNS) Generate(dependencies asset.Parents) error {
 			return err
 		}
 
-		//currently, this guesses the azure resource IDs from known parameter.
-		config.Spec.PublicZone = &configv1.DNSZone{
-			ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain),
+		if installConfig.Config.Publish == types.ExternalPublishingStrategy {
+			//currently, this guesses the azure resource IDs from known parameter.
+			config.Spec.PublicZone = &configv1.DNSZone{
+				ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain),
+			}
 		}
 		config.Spec.PrivateZone = &configv1.DNSZone{
 			ID: dnsConfig.GetPrivateDNSZoneID(clusterID.InfraID+"-rg", installConfig.Config.ClusterDomain()),

pkg/asset/manifests/openshift.go

@@ -19,6 +19,7 @@ import (
 	osmachine "github.com/openshift/installer/pkg/asset/machines/openstack"
 	"github.com/openshift/installer/pkg/asset/password"
 	"github.com/openshift/installer/pkg/asset/templates/content/openshift"
+	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 	azuretypes "github.com/openshift/installer/pkg/types/azure"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
@@ -55,6 +56,7 @@ func (o *Openshift) Dependencies() []asset.Asset {
 		&openshift.CloudCredsSecret{},
 		&openshift.KubeadminPasswordSecret{},
 		&openshift.RoleCloudCredsSecretReader{},
+		&openshift.PrivateClusterOutbound{},
 	}
 }
 
@@ -174,6 +176,12 @@ func (o *Openshift) Generate(dependencies asset.Parents) error {
 		assetData["99_role-cloud-creds-secret-reader.yaml"] = applyTemplateData(roleCloudCredsSecretReader.Files()[0].Data, templateData)
 	}
 
+	if platform == azuretypes.Name && installConfig.Config.Publish == types.InternalPublishingStrategy {
+		privateClusterOutbound := &openshift.PrivateClusterOutbound{}
+		dependencies.Get(privateClusterOutbound)
+		assetData["99_private-cluster-outbound-service.yaml"] = applyTemplateData(privateClusterOutbound.Files()[0].Data, templateData)
+	}
+
 	o.FileList = []*asset.File{}
 	for name, data := range assetData {
 		if len(data) == 0 {

pkg/asset/templates/content/openshift/private-cluster-outbound-service.go

@@ -0,0 +1,63 @@
+package openshift
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/templates/content"
+)
+
+const (
+	privateClusterOutboundFilename = "private-cluster-outbound-service.yaml"
+)
+
+var _ asset.WritableAsset = (*PrivateClusterOutbound)(nil)
+
+// PrivateClusterOutbound generates the private-cluster-outbound-*.yml files
+type PrivateClusterOutbound struct {
+	FileList []*asset.File
+}
+
+// Name returns a human friendly name for the asset.
+func (*PrivateClusterOutbound) Name() string {
+	return "Private Cluster Outbound Service"
+}
+
+// Dependencies returns all of the dependencies directly needed by the asset
+func (*PrivateClusterOutbound) Dependencies() []asset.Asset {
+	return []asset.Asset{}
+}
+
+// Generate generates the actual files by this asset
+func (p *PrivateClusterOutbound) Generate(dependencies asset.Parents) error {
+	data, err := content.GetOpenshiftTemplate(privateClusterOutboundFilename)
+	if err != nil {
+		return err
+	}
+
+	p.FileList = append(p.FileList, &asset.File{
+		Filename: filepath.Join(content.TemplateDir, privateClusterOutboundFilename),
+		Data:     []byte(data),
+	})
+	return nil
+}
+
+// Files returns the files generated by the asset.
+func (p *PrivateClusterOutbound) Files() []*asset.File {
+	return p.FileList
+}
+
+// Load returns the asset from disk
+func (p *PrivateClusterOutbound) Load(f asset.FileFetcher) (bool, error) {
+	file, err := f.FetchByName(filepath.Join(content.TemplateDir, privateClusterOutboundFilename))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	p.FileList = append(p.FileList, file)
+
+	return true, nil
+}

pkg/terraform/gather/azure/ip.go

@@ -12,17 +12,29 @@ import (
 
 // BootstrapIP returns the ip address for bootstrap host.
 func BootstrapIP(tfs *terraform.State) (string, error) {
+	var bootstrap string
+
 	publicIP, err := terraform.LookupResource(tfs, "module.bootstrap", "azurerm_public_ip", "bootstrap_public_ip")
+	if err == nil && len(publicIP.Instances) > 0 {
+		bootstrap, _, err = unstructured.NestedString(publicIP.Instances[0].Attributes, "ip_address")
+		if err != nil {
+			return "", errors.New("no public_ip found for bootstrap")
+		}
+		return bootstrap, nil
+	}
+
+	br, err := terraform.LookupResource(tfs, "module.bootstrap", "azurerm_network_interface", "bootstrap")
 	if err != nil {
-		return "", errors.Wrap(err, "failed to lookup public ip")
+		return "", errors.Wrap(err, "failed to lookup bootstrap network interface")
 	}
-	if len(publicIP.Instances) == 0 {
-		return "", errors.New("no public ip instance found")
+	if len(br.Instances) == 0 {
+		return "", errors.New("no bootstrap instance found")
 	}
-	bootstrap, _, err := unstructured.NestedString(publicIP.Instances[0].Attributes, "ip_address")
+	bootstrap, _, err = unstructured.NestedString(br.Instances[0].Attributes, "private_ip_address")
 	if err != nil {
-		return "", errors.New("no public_ip found for bootstrap")
+		return "", errors.New("no private_ip_address found for bootstrap")
 	}
+
 	return bootstrap, nil
 }