diff --git a/data/data/azure/bootstrap/main.tf b/data/data/azure/bootstrap/main.tf index 87d77b68e12..91293d72fd5 100644 --- a/data/data/azure/bootstrap/main.tf +++ b/data/data/azure/bootstrap/main.tf @@ -62,6 +62,8 @@ data "ignition_config" "redirect" { } resource "azurerm_public_ip" "bootstrap_public_ip" { + count = var.private ? 0 : 1 + sku = "Standard" location = var.region name = "${var.cluster_id}-bootstrap-pip" @@ -70,7 +72,9 @@ resource "azurerm_public_ip" "bootstrap_public_ip" { } data "azurerm_public_ip" "bootstrap_public_ip" { - name = azurerm_public_ip.bootstrap_public_ip.name + count = var.private ? 0 : 1 + + name = azurerm_public_ip.bootstrap_public_ip[0].name resource_group_name = var.resource_group_name } @@ -83,7 +87,7 @@ resource "azurerm_network_interface" "bootstrap" { subnet_id = var.subnet_id name = local.bootstrap_nic_ip_configuration_name private_ip_address_allocation = "Dynamic" - public_ip_address_id = azurerm_public_ip.bootstrap_public_ip.id + public_ip_address_id = var.private ? null : azurerm_public_ip.bootstrap_public_ip[0].id } } diff --git a/data/data/azure/bootstrap/variables.tf b/data/data/azure/bootstrap/variables.tf index b4eaa9d6e3f..445182b1a48 100644 --- a/data/data/azure/bootstrap/variables.tf +++ b/data/data/azure/bootstrap/variables.tf @@ -63,3 +63,8 @@ variable "nsg_name" { type = string description = "The network security group for the subnet." } + +variable "private" { + type = bool + description = "This value determines if this is a private cluster or not." +} diff --git a/data/data/azure/dns/dns.tf b/data/data/azure/dns/dns.tf index 51c30b87558..0fe7e3e093f 100644 --- a/data/data/azure/dns/dns.tf +++ b/data/data/azure/dns/dns.tf @@ -32,6 +32,8 @@ resource "azureprivatedns_a_record" "api_internal" { } resource "azurerm_dns_cname_record" "api_external" { + count = var.private ? 0 : 1 + name = local.api_external_name zone_name = var.base_domain resource_group_name = var.base_domain_resource_group_name diff --git a/data/data/azure/dns/variables.tf b/data/data/azure/dns/variables.tf index d62176e80a4..41f26f45d81 100644 --- a/data/data/azure/dns/variables.tf +++ b/data/data/azure/dns/variables.tf @@ -54,3 +54,8 @@ variable "resource_group_name" { type = string description = "Resource group for the deployment" } + +variable "private" { + type = bool + description = "This value determines if this is a private cluster or not." +} diff --git a/data/data/azure/main.tf b/data/data/azure/main.tf index 5a00703270f..ea3553da5d5 100644 --- a/data/data/azure/main.tf +++ b/data/data/azure/main.tf @@ -36,6 +36,7 @@ module "bootstrap" { tags = local.tags storage_account = azurerm_storage_account.cluster nsg_name = module.vnet.master_nsg_name + private = module.vnet.private } module "vnet" { @@ -51,6 +52,7 @@ module "vnet" { virtual_network_name = var.azure_virtual_network master_subnet = var.azure_control_plane_subnet worker_subnet = var.azure_compute_subnet + private = var.azure_private } module "master" { @@ -71,6 +73,7 @@ module "master" { storage_account = azurerm_storage_account.cluster os_volume_type = var.azure_master_root_volume_type os_volume_size = var.azure_master_root_volume_size + private = module.vnet.private } module "dns" { @@ -85,6 +88,7 @@ module "dns" { base_domain_resource_group_name = var.azure_base_domain_resource_group_name etcd_count = var.master_count etcd_ip_addresses = module.master.ip_addresses + private = module.vnet.private } resource "random_string" "storage_suffix" { diff --git a/data/data/azure/master/master.tf b/data/data/azure/master/master.tf index c1d420aa3ee..861966d3cb6 100644 --- a/data/data/azure/master/master.tf +++ b/data/data/azure/master/master.tf @@ -19,21 +19,24 @@ resource "azurerm_network_interface" "master" { } resource "azurerm_network_interface_backend_address_pool_association" "master" { - count = var.instance_count + count = var.instance_count + network_interface_id = element(azurerm_network_interface.master.*.id, count.index) backend_address_pool_id = var.elb_backend_pool_id ip_configuration_name = local.ip_configuration_name #must be the same as nic's ip configuration name. } resource "azurerm_network_interface_backend_address_pool_association" "master_internal" { - count = var.instance_count + count = var.instance_count + network_interface_id = element(azurerm_network_interface.master.*.id, count.index) backend_address_pool_id = var.ilb_backend_pool_id ip_configuration_name = local.ip_configuration_name #must be the same as nic's ip configuration name. } resource "azurerm_virtual_machine" "master" { - count = var.instance_count + count = var.instance_count + name = "${var.cluster_id}-master-${count.index}" location = var.region zones = compact([var.availability_zones[count.index]]) diff --git a/data/data/azure/master/variables.tf b/data/data/azure/master/variables.tf index d56a6a0c4c4..255a49b0e70 100644 --- a/data/data/azure/master/variables.tf +++ b/data/data/azure/master/variables.tf @@ -86,3 +86,8 @@ variable "availability_zones" { type = list(string) description = "List of the availability zones in which to create the masters. The length of this list must match instance_count." } + +variable "private" { + type = bool + description = "This value determines if this is a private cluster or not." +} diff --git a/data/data/azure/variables-azure.tf b/data/data/azure/variables-azure.tf index 25495241ef9..33bbd0b50b8 100644 --- a/data/data/azure/variables-azure.tf +++ b/data/data/azure/variables-azure.tf @@ -106,3 +106,8 @@ variable "azure_compute_subnet" { type = string description = "The name of the subnet for worker nodes, either existing or to be created" } + +variable "azure_private" { + type = bool + description = "This determines if this is a private cluster or not." +} diff --git a/data/data/azure/vnet/outputs.tf b/data/data/azure/vnet/outputs.tf index 994eab1de3c..6357f20d6d1 100644 --- a/data/data/azure/vnet/outputs.tf +++ b/data/data/azure/vnet/outputs.tf @@ -1,5 +1,5 @@ output "cluster-pip" { - value = azurerm_public_ip.cluster_public_ip.ip_address + value = var.private ? null : azurerm_public_ip.cluster_public_ip.ip_address } output "public_lb_backend_pool_id" { @@ -11,11 +11,11 @@ output "internal_lb_backend_pool_id" { } output "public_lb_id" { - value = azurerm_lb.public.id + value = var.private ? null : azurerm_lb.public.id } output "public_lb_pip_fqdn" { - value = data.azurerm_public_ip.cluster_public_ip.fqdn + value = var.private ? null : data.azurerm_public_ip.cluster_public_ip.fqdn } output "internal_lb_ip_address" { @@ -37,3 +37,7 @@ output "master_subnet_id" { output "worker_subnet_id" { value = local.worker_subnet_id } + +output "private" { + value = var.private +} diff --git a/data/data/azure/vnet/public-lb.tf b/data/data/azure/vnet/public-lb.tf index 86abe0772a0..896b59c5d71 100644 --- a/data/data/azure/vnet/public-lb.tf +++ b/data/data/azure/vnet/public-lb.tf @@ -35,6 +35,8 @@ resource "azurerm_lb_backend_address_pool" "master_public_lb_pool" { } resource "azurerm_lb_rule" "public_lb_rule_api_internal" { + count = var.private ? 0 : 1 + name = "api-internal" resource_group_name = var.resource_group_name protocol = "Tcp" @@ -46,10 +48,28 @@ resource "azurerm_lb_rule" "public_lb_rule_api_internal" { enable_floating_ip = false idle_timeout_in_minutes = 30 load_distribution = "Default" - probe_id = azurerm_lb_probe.public_lb_probe_api_internal.id + probe_id = azurerm_lb_probe.public_lb_probe_api_internal[0].id +} + +resource "azurerm_lb_rule" "internal_outbound_rule" { + count = var.private ? 1 : 0 + + name = "internal_outbound_rule" + resource_group_name = var.resource_group_name + protocol = "Tcp" + backend_address_pool_id = azurerm_lb_backend_address_pool.master_public_lb_pool.id + loadbalancer_id = azurerm_lb.public.id + frontend_port = 27627 + backend_port = 27627 + frontend_ip_configuration_name = local.public_lb_frontend_ip_configuration_name + enable_floating_ip = false + idle_timeout_in_minutes = 30 + load_distribution = "Default" } resource "azurerm_lb_probe" "public_lb_probe_api_internal" { + count = var.private ? 0 : 1 + name = "api-internal-probe" resource_group_name = var.resource_group_name interval_in_seconds = 10 diff --git a/data/data/azure/vnet/variables.tf b/data/data/azure/vnet/variables.tf index 19e3b9f3fdc..929ddc7d0ca 100644 --- a/data/data/azure/vnet/variables.tf +++ b/data/data/azure/vnet/variables.tf @@ -52,3 +52,8 @@ variable "worker_subnet" { type = string description = "This is the name of the subnet used for the compute nodes, new or existing" } + +variable "private" { + type = bool + description = "The determines if this is a private/internal cluster or not." +} diff --git a/data/data/manifests/openshift/private-cluster-outbound-service.yaml b/data/data/manifests/openshift/private-cluster-outbound-service.yaml new file mode 100644 index 00000000000..14c7dea1117 --- /dev/null +++ b/data/data/manifests/openshift/private-cluster-outbound-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + namespace: openshift-config-managed + name: outbound-provider +spec: + type: LoadBalancer + ports: + - port: 27627 diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index c919aa262c3..91cc5d7fe34 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -223,6 +223,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error { WorkerConfigs: workerConfigs, ImageURL: string(*rhcosImage), PreexistingNetwork: preexistingnetwork, + Publish: installConfig.Config.Publish, }, ) if err != nil { diff --git a/pkg/asset/manifests/cloudproviderconfig.go b/pkg/asset/manifests/cloudproviderconfig.go index 04918d4370f..c2706a8afb5 100644 --- a/pkg/asset/manifests/cloudproviderconfig.go +++ b/pkg/asset/manifests/cloudproviderconfig.go @@ -112,7 +112,6 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error { if installConfig.Config.Azure.ComputeSubnet != "" { subnet = installConfig.Config.Azure.ComputeSubnet } - azureConfig, err := azure.CloudProviderConfig{ GroupLocation: installConfig.Config.Azure.Region, ResourcePrefix: clusterID.InfraID, diff --git a/pkg/asset/manifests/dns.go b/pkg/asset/manifests/dns.go index 045523869d1..600a45d5d72 100644 --- a/pkg/asset/manifests/dns.go +++ b/pkg/asset/manifests/dns.go @@ -96,9 +96,11 @@ func (d *DNS) Generate(dependencies asset.Parents) error { return err } - //currently, this guesses the azure resource IDs from known parameter. - config.Spec.PublicZone = &configv1.DNSZone{ - ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain), + if installConfig.Config.Publish == types.ExternalPublishingStrategy { + //currently, this guesses the azure resource IDs from known parameter. + config.Spec.PublicZone = &configv1.DNSZone{ + ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain), + } } config.Spec.PrivateZone = &configv1.DNSZone{ ID: dnsConfig.GetPrivateDNSZoneID(clusterID.InfraID+"-rg", installConfig.Config.ClusterDomain()), diff --git a/pkg/asset/manifests/openshift.go b/pkg/asset/manifests/openshift.go index fb8051ad4b4..aabcef33be0 100644 --- a/pkg/asset/manifests/openshift.go +++ b/pkg/asset/manifests/openshift.go @@ -19,6 +19,7 @@ import ( osmachine "github.com/openshift/installer/pkg/asset/machines/openstack" "github.com/openshift/installer/pkg/asset/password" "github.com/openshift/installer/pkg/asset/templates/content/openshift" + "github.com/openshift/installer/pkg/types" awstypes "github.com/openshift/installer/pkg/types/aws" azuretypes "github.com/openshift/installer/pkg/types/azure" gcptypes "github.com/openshift/installer/pkg/types/gcp" @@ -55,6 +56,7 @@ func (o *Openshift) Dependencies() []asset.Asset { &openshift.CloudCredsSecret{}, &openshift.KubeadminPasswordSecret{}, &openshift.RoleCloudCredsSecretReader{}, + &openshift.PrivateClusterOutbound{}, } } @@ -174,6 +176,12 @@ func (o *Openshift) Generate(dependencies asset.Parents) error { assetData["99_role-cloud-creds-secret-reader.yaml"] = applyTemplateData(roleCloudCredsSecretReader.Files()[0].Data, templateData) } + if platform == azuretypes.Name && installConfig.Config.Publish == types.InternalPublishingStrategy { + privateClusterOutbound := &openshift.PrivateClusterOutbound{} + dependencies.Get(privateClusterOutbound) + assetData["99_private-cluster-outbound-service.yaml"] = applyTemplateData(privateClusterOutbound.Files()[0].Data, templateData) + } + o.FileList = []*asset.File{} for name, data := range assetData { if len(data) == 0 { diff --git a/pkg/asset/templates/content/openshift/private-cluster-outbound-service.go b/pkg/asset/templates/content/openshift/private-cluster-outbound-service.go new file mode 100644 index 00000000000..a8d98afc7b4 --- /dev/null +++ b/pkg/asset/templates/content/openshift/private-cluster-outbound-service.go @@ -0,0 +1,63 @@ +package openshift + +import ( + "os" + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/templates/content" +) + +const ( + privateClusterOutboundFilename = "private-cluster-outbound-service.yaml" +) + +var _ asset.WritableAsset = (*PrivateClusterOutbound)(nil) + +// PrivateClusterOutbound generates the private-cluster-outbound-*.yml files +type PrivateClusterOutbound struct { + FileList []*asset.File +} + +// Name returns a human friendly name for the asset. +func (*PrivateClusterOutbound) Name() string { + return "Private Cluster Outbound Service" +} + +// Dependencies returns all of the dependencies directly needed by the asset +func (*PrivateClusterOutbound) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +// Generate generates the actual files by this asset +func (p *PrivateClusterOutbound) Generate(dependencies asset.Parents) error { + data, err := content.GetOpenshiftTemplate(privateClusterOutboundFilename) + if err != nil { + return err + } + + p.FileList = append(p.FileList, &asset.File{ + Filename: filepath.Join(content.TemplateDir, privateClusterOutboundFilename), + Data: []byte(data), + }) + return nil +} + +// Files returns the files generated by the asset. +func (p *PrivateClusterOutbound) Files() []*asset.File { + return p.FileList +} + +// Load returns the asset from disk +func (p *PrivateClusterOutbound) Load(f asset.FileFetcher) (bool, error) { + file, err := f.FetchByName(filepath.Join(content.TemplateDir, privateClusterOutboundFilename)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + p.FileList = append(p.FileList, file) + + return true, nil +} diff --git a/pkg/terraform/gather/azure/ip.go b/pkg/terraform/gather/azure/ip.go index 0ee792608f7..6520f6d2af7 100644 --- a/pkg/terraform/gather/azure/ip.go +++ b/pkg/terraform/gather/azure/ip.go @@ -12,17 +12,29 @@ import ( // BootstrapIP returns the ip address for bootstrap host. func BootstrapIP(tfs *terraform.State) (string, error) { + var bootstrap string + publicIP, err := terraform.LookupResource(tfs, "module.bootstrap", "azurerm_public_ip", "bootstrap_public_ip") + if err == nil && len(publicIP.Instances) > 0 { + bootstrap, _, err = unstructured.NestedString(publicIP.Instances[0].Attributes, "ip_address") + if err != nil { + return "", errors.New("no public_ip found for bootstrap") + } + return bootstrap, nil + } + + br, err := terraform.LookupResource(tfs, "module.bootstrap", "azurerm_network_interface", "bootstrap") if err != nil { - return "", errors.Wrap(err, "failed to lookup public ip") + return "", errors.Wrap(err, "failed to lookup bootstrap network interface") } - if len(publicIP.Instances) == 0 { - return "", errors.New("no public ip instance found") + if len(br.Instances) == 0 { + return "", errors.New("no bootstrap instance found") } - bootstrap, _, err := unstructured.NestedString(publicIP.Instances[0].Attributes, "ip_address") + bootstrap, _, err = unstructured.NestedString(br.Instances[0].Attributes, "private_ip_address") if err != nil { - return "", errors.New("no public_ip found for bootstrap") + return "", errors.New("no private_ip_address found for bootstrap") } + return bootstrap, nil } diff --git a/pkg/tfvars/azure/azure.go b/pkg/tfvars/azure/azure.go index b7b289e431f..b8e2a98d8ec 100644 --- a/pkg/tfvars/azure/azure.go +++ b/pkg/tfvars/azure/azure.go @@ -5,6 +5,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" + "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/azure/defaults" azureprovider "sigs.k8s.io/cluster-api-provider-azure/pkg/apis/azureprovider/v1beta1" ) @@ -33,6 +34,7 @@ type config struct { ControlPlaneSubnet string `json:"azure_control_plane_subnet"` ComputeSubnet string `json:"azure_compute_subnet"` PreexistingNetwork bool `json:"azure_preexisting_network"` + Private bool `json:"azure_private"` } // TFVarsSources contains the parameters to be converted into Terraform variables @@ -43,6 +45,7 @@ type TFVarsSources struct { WorkerConfigs []*azureprovider.AzureMachineProviderSpec ImageURL string PreexistingNetwork bool + Publish types.PublishingStrategy } // TFVars generates Azure-specific Terraform variables launching the cluster. @@ -66,6 +69,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) { VolumeType: masterConfig.OSDisk.ManagedDisk.StorageAccountType, VolumeSize: masterConfig.OSDisk.DiskSizeGB, ImageURL: sources.ImageURL, + Private: sources.Publish == types.InternalPublishingStrategy, BaseDomainResourceGroupName: sources.BaseDomainResourceGroupName, NetworkResourceGroupName: masterConfig.NetworkResourceGroup, VirtualNetwork: masterConfig.Vnet, diff --git a/pkg/types/azure/validation/platform.go b/pkg/types/azure/validation/platform.go index bafc603c0c4..304f687a3d0 100644 --- a/pkg/types/azure/validation/platform.go +++ b/pkg/types/azure/validation/platform.go @@ -1,18 +1,21 @@ package validation import ( + "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/azure" "k8s.io/apimachinery/pkg/util/validation/field" ) // ValidatePlatform checks that the specified platform is valid. -func ValidatePlatform(p *azure.Platform, fldPath *field.Path) field.ErrorList { +func ValidatePlatform(p *azure.Platform, publish types.PublishingStrategy, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if p.Region == "" { allErrs = append(allErrs, field.Required(fldPath.Child("region"), "region should be set to one of the supported Azure regions")) } - if p.BaseDomainResourceGroupName == "" { - allErrs = append(allErrs, field.Required(fldPath.Child("baseDomainResourceGroupName"), "baseDomainResourceGroupName is the resource group name where the azure dns zone is deployed")) + if publish == types.ExternalPublishingStrategy { + if p.BaseDomainResourceGroupName == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("baseDomainResourceGroupName"), "baseDomainResourceGroupName is the resource group name where the azure dns zone is deployed")) + } } if p.DefaultMachinePlatform != nil { allErrs = append(allErrs, ValidateMachinePool(p.DefaultMachinePlatform, fldPath.Child("defaultMachinePlatform"))...) diff --git a/pkg/types/azure/validation/platform_test.go b/pkg/types/azure/validation/platform_test.go index d6fda7c75a2..73015df86bf 100644 --- a/pkg/types/azure/validation/platform_test.go +++ b/pkg/types/azure/validation/platform_test.go @@ -3,6 +3,7 @@ package validation import ( "testing" + "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/azure" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/validation/field" @@ -50,7 +51,7 @@ func TestValidatePlatform(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - err := ValidatePlatform(tc.platform, field.NewPath("test-path")).ToAggregate() + err := ValidatePlatform(tc.platform, types.ExternalPublishingStrategy, field.NewPath("test-path")).ToAggregate() if tc.valid { assert.NoError(t, err) } else { diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go index b597cf27160..d2b79afcc56 100644 --- a/pkg/types/validation/installconfig.go +++ b/pkg/types/validation/installconfig.go @@ -225,7 +225,9 @@ func validatePlatform(platform *types.Platform, fldPath *field.Path, openStackVa validate(aws.Name, platform.AWS, func(f *field.Path) field.ErrorList { return awsvalidation.ValidatePlatform(platform.AWS, f) }) } if platform.Azure != nil { - validate(azure.Name, platform.Azure, func(f *field.Path) field.ErrorList { return azurevalidation.ValidatePlatform(platform.Azure, f) }) + validate(azure.Name, platform.Azure, func(f *field.Path) field.ErrorList { + return azurevalidation.ValidatePlatform(platform.Azure, c.Publish, f) + }) } if platform.GCP != nil { validate(gcp.Name, platform.GCP, func(f *field.Path) field.ErrorList { return gcpvalidation.ValidatePlatform(platform.GCP, f) })