CORS-3195: Azure CAPI Supplemental Load Balancers

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.0.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.2
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1

go.sum

@@ -1192,6 +1192,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 h1:z4Yei
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0/go.mod h1:rko9SzMxcMk0NJsNAxALEGaTYyy79bNRwxgJfrH0Spw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.0.0 h1:nBy98uKOIfun5z6wx6jwWLrULcM0+cjBalBFZlEZ7CA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.0.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 h1:9Eih8XcEeQnFD0ntMlUDleKMzfeCeUfa+VbnDCI4AZs=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0/go.mod h1:wGPyTi+aURdqPAGMZDQqnNs9IrShADF8w2WZb6bKeq0=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.2 h1:f9lam+D19V0TDn17+aFhrVhWPpfsF5zaGHeqDGJZAVc=

pkg/infrastructure/azure/azure.go

@@ -3,7 +3,6 @@ package azure
 import (
 	"context"
 	"fmt"
-	"log"
 	"math/rand"
 	"net/http"
 	"strings"
@@ -13,31 +12,37 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
 	"github.com/coreos/stream-metadata-go/arch"
 	"github.com/sirupsen/logrus"
 	"k8s.io/utils/ptr"
+	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
 	"github.com/openshift/installer/pkg/rhcos"
+	"github.com/openshift/installer/pkg/types"
 	aztypes "github.com/openshift/installer/pkg/types/azure"
 )
 
 // Provider implements Azure CAPI installation.
 type Provider struct {
-	clusterapi.InfraProvider
 	ResourceGroupName    string
 	StorageAccountName   string
 	StorageURL           string
 	StorageAccount       *armstorage.Account
 	StorageClientFactory *armstorage.ClientFactory
 	StorageAccountKeys   []armstorage.AccountKey
 	Tags                 map[string]*string
+	lbBackendAddressPool *armnetwork.BackendAddressPool
 }
 
 var _ clusterapi.PreProvider = (*Provider)(nil)
 var _ clusterapi.InfraReadyProvider = (*Provider)(nil)
+var _ clusterapi.PostProvider = (*Provider)(nil)
 
 // Name returns the name of the provider.
 func (p *Provider) Name() string {
@@ -158,7 +163,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 	userAssignedIdentityName := fmt.Sprintf("%s-identity", in.InfraID)
 	armmsiClientFactory, err := armmsi.NewClientFactory(subscriptionID, tokenCredential, nil)
 	if err != nil {
-		log.Fatalf("failed to create armmsi client: %v", err)
+		return fmt.Errorf("failed to create armmsi client: %w", err)
 	}
 	userAssignedIdentity, err := armmsiClientFactory.NewUserAssignedIdentitiesClient().CreateOrUpdate(
 		ctx,
@@ -171,7 +176,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		nil,
 	)
 	if err != nil {
-		log.Fatalf("failed to create user assigned identity %s: %v", userAssignedIdentityName, err)
+		return fmt.Errorf("failed to create user assigned identity %s: %w", userAssignedIdentityName, err)
 	}
 	principalID := *userAssignedIdentity.Properties.PrincipalID
 
@@ -316,15 +321,141 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		return err
 	}
 
+	networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, session.TokenCreds, nil)
+	if err != nil {
+		return fmt.Errorf("error creating network client factory: %w", err)
+	}
+
+	lbClient := networkClientFactory.NewLoadBalancersClient()
+
+	lbInput := &lbInput{
+		infraID:        in.InfraID,
+		region:         in.InstallConfig.Config.Azure.Region,
+		resourceGroup:  resourceGroupName,
+		subscriptionID: session.Credentials.SubscriptionID,
+		lbClient:       lbClient,
+		pipClient:      networkClientFactory.NewPublicIPAddressesClient(),
+	}
+
+	intLoadBalancer, err := updateInternalLoadBalancer(ctx, lbInput)
+	if err != nil {
+		return fmt.Errorf("failed to update internal load balancer: %w", err)
+	}
+	logrus.Debugf("updated internal load balancer: %s", *intLoadBalancer.ID)
+
+	var lbBap *armnetwork.BackendAddressPool
+	var extLBFQDN string
+	if in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy {
+		publicIP, err := createPublicIP(ctx, lbInput)
+		if err != nil {
+			return fmt.Errorf("failed to create public ip: %w", err)
+		}
+		logrus.Debugf("created public ip: %s", *publicIP.ID)
+
+		loadBalancer, err := createExternalLoadBalancer(ctx, publicIP, lbInput)
+		if err != nil {
+			return fmt.Errorf("failed to create load balancer: %w", err)
+		}
+		logrus.Debugf("created load balancer: %s", *loadBalancer.ID)
+		lbBap = loadBalancer.Properties.BackendAddressPools[0]
+		extLBFQDN = *publicIP.Properties.DNSSettings.Fqdn
+	}
+
 	// Save context for other hooks
 	p.ResourceGroupName = resourceGroupName
 	p.StorageAccountName = storageAccountName
 	p.StorageURL = storageURL
 	p.StorageAccount = storageAccount
 	p.StorageClientFactory = storageClientFactory
 	p.StorageAccountKeys = storageAccountKeys
+	p.lbBackendAddressPool = lbBap
+
+	if err := createDNSEntries(ctx, in, extLBFQDN); err != nil {
+		return fmt.Errorf("error creating DNS records: %w", err)
+	}
+
+	return nil
+}
+
+// PostProvision provisions an external Load Balancer (when appropriate), and adds configuration
+// for the MCS to the CAPI-provisioned internal LB.
+func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisionInput) error {
+	ssn, err := in.InstallConfig.Azure.Session()
+	if err != nil {
+		return fmt.Errorf("error retrieving Azure session: %w", err)
+	}
+	subscriptionID := ssn.Credentials.SubscriptionID
+
+	if in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy {
+		vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, ssn.TokenCreds, nil)
+		if err != nil {
+			return fmt.Errorf("error creating vm client: %w", err)
+		}
+		nicClient, err := armnetwork.NewInterfacesClient(ssn.Credentials.SubscriptionID, ssn.TokenCreds, nil)
+		if err != nil {
+			return fmt.Errorf("error creating nic client: %w", err)
+		}
+
+		vmIDs, err := getControlPlaneIDs(in.Client, in.InstallConfig.Config.ControlPlane.Replicas, in.InfraID)
+		if err != nil {
+			return fmt.Errorf("failed to get control plane VM IDs: %w", err)
+		}
+
+		vmInput := &vmInput{
+			infraID:       in.InfraID,
+			resourceGroup: p.ResourceGroupName,
+			vmClient:      vmClient,
+			nicClient:     nicClient,
+			ids:           vmIDs,
+			bap:           p.lbBackendAddressPool,
+		}
+
+		if err = associateVMToBackendPool(ctx, *vmInput); err != nil {
+			return fmt.Errorf("failed to associate control plane VMs with external load balancer: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func getControlPlaneIDs(cl client.Client, replicas *int64, infraID string) ([]string, error) {
+	res := []string{}
+	total := int64(1)
+	if replicas != nil {
+		total = *replicas
+	}
+	for i := int64(0); i < total; i++ {
+		machineName := fmt.Sprintf("%s-master-%d", infraID, i)
+		key := client.ObjectKey{
+			Name:      machineName,
+			Namespace: capiutils.Namespace,
+		}
+		azureMachine := &capz.AzureMachine{}
+		if err := cl.Get(context.Background(), key, azureMachine); err != nil {
+			return nil, fmt.Errorf("failed to get AzureMachine: %w", err)
+		}
+		if vmID := azureMachine.Spec.ProviderID; vmID != nil && len(*vmID) != 0 {
+			res = append(res, *azureMachine.Spec.ProviderID)
+		} else {
+			return nil, fmt.Errorf("%s .Spec.ProviderID is empty", machineName)
+		}
+	}
 
-	return createDNSEntries(ctx, in)
+	bootstrapName := capiutils.GenerateBoostrapMachineName(infraID)
+	key := client.ObjectKey{
+		Name:      bootstrapName,
+		Namespace: capiutils.Namespace,
+	}
+	azureMachine := &capz.AzureMachine{}
+	if err := cl.Get(context.Background(), key, azureMachine); err != nil {
+		return nil, fmt.Errorf("failed to get AzureMachine: %w", err)
+	}
+	if vmID := azureMachine.Spec.ProviderID; vmID != nil && len(*vmID) != 0 {
+		res = append(res, *azureMachine.Spec.ProviderID)
+	} else {
+		return nil, fmt.Errorf("%s .Spec.ProviderID is empty", bootstrapName)
+	}
+	return res, nil
 }
 
 func randomString(length int) string {

pkg/infrastructure/azure/dns.go

@@ -37,7 +37,7 @@ type recordPrivateList struct {
 }
 
 // Create DNS entries for azure.
-func createDNSEntries(ctx context.Context, in clusterapi.InfraReadyInput) error {
+func createDNSEntries(ctx context.Context, in clusterapi.InfraReadyInput, extLBFQDN string) error {
 	private := in.InstallConfig.Config.Publish == types.InternalPublishingStrategy
 	baseDomainResourceGroup := in.InstallConfig.Config.Azure.BaseDomainResourceGroupName
 	zone := in.InstallConfig.Config.BaseDomain
@@ -107,8 +107,7 @@ func createDNSEntries(ctx context.Context, in clusterapi.InfraReadyInput) error
 		// if useIPv6 {
 		// 	cnameRecordName = apiExternalNameV6
 		// }
-		// TODO: Populate with public LB FQDN. Placeholder text as value.
-		publicRecords := createRecordSet(cnameRecordName, azureTags, ttl, cname, "", in.InstallConfig.Config.ClusterDomain())
+		publicRecords := createRecordSet(cnameRecordName, azureTags, ttl, cname, "", extLBFQDN)
 		_, err = recordSetClient.CreateOrUpdate(ctx, baseDomainResourceGroup, zone, publicRecords.Name, publicRecords.RecordType, publicRecords.RecordSet, nil)
 		if err != nil {
 			return fmt.Errorf("failed to create public record set: %w", err)