SPLAT-1141: implement vSphere control plane machinesets

go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/microsoftgraph/msgraph-sdk-go v0.47.0
 	github.com/nutanix-cloud-native/prism-go-client v0.2.1-0.20220804130801-c8a253627c64
 	github.com/onsi/gomega v1.28.1
-	github.com/openshift/api v0.0.0-20231019121755-0c5924392281
+	github.com/openshift/api v0.0.0-20231024112103-79b9cd5e6020
 	github.com/openshift/assisted-image-service v0.0.0-20230829160050-0b98ec74397b
 	github.com/openshift/assisted-service/api v0.0.0
 	github.com/openshift/assisted-service/client v0.0.0

go.sum

@@ -989,8 +989,8 @@ github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xA
 github.com/openshift/api v0.0.0-20200326160804-ecb9283fe820/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE=
 github.com/openshift/api v0.0.0-20200827090112-c05698d102cf/go.mod h1:M3xexPhgM8DISzzRpuFUy+jfPjQPIcs9yqEYj17mXV8=
 github.com/openshift/api v0.0.0-20200829102639-8a3a835f1acf/go.mod h1:M3xexPhgM8DISzzRpuFUy+jfPjQPIcs9yqEYj17mXV8=
-github.com/openshift/api v0.0.0-20231019121755-0c5924392281 h1:49ccho6DQLZpqxlFWyXpHRoLsQl0KVma+Ir+2zC9qSY=
-github.com/openshift/api v0.0.0-20231019121755-0c5924392281/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
+github.com/openshift/api v0.0.0-20231024112103-79b9cd5e6020 h1:oJfieDRL/5RJLu11oSpgZsI8EmGniTrQn/SyC6BZc68=
+github.com/openshift/api v0.0.0-20231024112103-79b9cd5e6020/go.mod h1:qNtV0315F+f8ld52TLtPvrfivZpdimOzTi3kn9IVbtU=
 github.com/openshift/assisted-image-service v0.0.0-20230829160050-0b98ec74397b h1:wLVEgmzQjs3t4Z96gZzSLF/ws6ULliAks7z1lozNJrE=
 github.com/openshift/assisted-image-service v0.0.0-20230829160050-0b98ec74397b/go.mod h1:KTt/pnfs9gt0McDPrb0zVTkwd0xIFNik/ZJROIBzsbc=
 github.com/openshift/assisted-service/api v0.0.0-20230831114549-1922eda29cf8 h1:+fZLKbycDo4JeLwPGVSAgf2XPaJGLM341l9ZfrrlxG0=

pkg/asset/machines/aws/machines.go

@@ -149,7 +149,7 @@ func Machines(clusterID string, region string, subnets map[string]string, pool *
 			Template: machinev1.ControlPlaneMachineSetTemplate{
 				MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
 				OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
-					FailureDomains: machinev1.FailureDomains{
+					FailureDomains: &machinev1.FailureDomains{
 						Platform: v1.AWSPlatformType,
 						AWS:      &failureDomains,
 					},

pkg/asset/machines/azure/machines.go

@@ -142,7 +142,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 	}
 
 	if len(failureDomains) > 0 {
-		controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = machinev1.FailureDomains{
+		controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = &machinev1.FailureDomains{
 			Platform: v1.AzurePlatformType,
 			Azure:    &failureDomains,
 		}

pkg/asset/machines/gcp/machines.go

@@ -106,7 +106,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 			Template: machinev1.ControlPlaneMachineSetTemplate{
 				MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
 				OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
-					FailureDomains: machinev1.FailureDomains{
+					FailureDomains: &machinev1.FailureDomains{
 						Platform: v1.GCPPlatformType,
 						GCP:      &failureDomains,
 					},

pkg/asset/machines/master.go

@@ -473,10 +473,15 @@ func (m *Master) Generate(dependencies asset.Parents) error {
 		pool.Platform.VSphere = &mpool
 		templateName := clusterID.InfraID + "-rhcos"
 
-		machines, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName)
+		machines, controlPlaneMachineSet, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName)
 		if err != nil {
 			return errors.Wrap(err, "failed to create master machine objects")
 		}
+
+		if ic.FeatureSet != configv1.TechPreviewNoUpgrade {
+			controlPlaneMachineSet = nil
+		}
+
 		vsphere.ConfigMasters(machines, clusterID.InfraID)
 	case powervstypes.Name:
 		mpool := defaultPowerVSMachinePoolPlatform()

pkg/asset/machines/openstack/machines.go

@@ -148,7 +148,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 	}
 
 	if CPMSFailureDomains := pruneFailureDomains(failureDomains); CPMSFailureDomains != nil {
-		controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = machinev1.FailureDomains{
+		controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains = &machinev1.FailureDomains{
 			Platform:  v1.OpenStackPlatformType,
 			OpenStack: CPMSFailureDomains,
 		}

pkg/asset/machines/vsphere/machines.go

@@ -10,35 +10,37 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 
+	v1 "github.com/openshift/api/config/v1"
+	machinev1 "github.com/openshift/api/machine/v1"
 	machineapi "github.com/openshift/api/machine/v1beta1"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/vsphere"
 )
 
 // Machines returns a list of machines for a machinepool.
-func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string) ([]machineapi.Machine, error) {
+func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string) ([]machineapi.Machine, *machinev1.ControlPlaneMachineSet, error) {
 	if configPlatform := config.Platform.Name(); configPlatform != vsphere.Name {
-		return nil, fmt.Errorf("non vsphere configuration: %q", configPlatform)
+		return nil, nil, fmt.Errorf("non vsphere configuration: %q", configPlatform)
 	}
 	if poolPlatform := pool.Platform.Name(); poolPlatform != vsphere.Name {
-		return nil, fmt.Errorf("non-VSphere machine-pool: %q", poolPlatform)
+		return nil, nil, fmt.Errorf("non-VSphere machine-pool: %q", poolPlatform)
 	}
 
 	var failureDomain vsphere.FailureDomain
 	var machines []machineapi.Machine
 	platform := config.Platform.VSphere
 	mpool := pool.Platform.VSphere
-	replicas := int64(1)
+	replicas := int32(1)
 
 	numOfZones := len(mpool.Zones)
 
 	zones, err := getDefinedZonesFromTopology(platform)
 	if err != nil {
-		return machines, err
+		return machines, nil, err
 	}
 
 	if pool.Replicas != nil {
-		replicas = *pool.Replicas
+		replicas = int32(*pool.Replicas)
 	}
 
 	// Create hosts to populate from.  Copying so we can remove without changing original
@@ -53,7 +55,11 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 		}
 	}
 
-	for idx := int64(0); idx < replicas; idx++ {
+	failureDomains := []machinev1.VSphereFailureDomain{}
+
+	vsphereMachineProvider := &machineapi.VSphereMachineProviderSpec{}
+
+	for idx := int32(0); idx < replicas; idx++ {
 		logrus.Debugf("Creating %v machine %v", role, idx)
 		var host *vsphere.Host
 		desiredZone := mpool.Zones[int(idx)%numOfZones]
@@ -66,7 +72,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 		logrus.Debugf("Desired zone: %v", desiredZone)
 
 		if _, exists := zones[desiredZone]; !exists {
-			return nil, errors.Errorf("zone [%s] specified by machinepool is not defined", desiredZone)
+			return nil, nil, errors.Errorf("zone [%s] specified by machinepool is not defined", desiredZone)
 		}
 
 		failureDomain = zones[desiredZone]
@@ -77,18 +83,22 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 			"machine.openshift.io/cluster-api-machine-type": role,
 		}
 
+		failureDomains = append(failureDomains, machinev1.VSphereFailureDomain{
+			Name: failureDomain.Name,
+		})
+
 		osImageForZone := failureDomain.Topology.Template
 		if failureDomain.Topology.Template == "" {
 			osImageForZone = fmt.Sprintf("%s-%s-%s", osImage, failureDomain.Region, failureDomain.Zone)
 		}
 
 		vcenter, err := getVCenterFromServerName(failureDomain.Server, platform)
 		if err != nil {
-			return nil, errors.Wrap(err, "unable to find vCenter in failure domains")
+			return nil, nil, errors.Wrap(err, "unable to find vCenter in failure domains")
 		}
 		provider, err := provider(clusterID, vcenter, failureDomain, mpool, osImageForZone, userDataSecret)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to create provider")
+			return nil, nil, errors.Wrap(err, "failed to create provider")
 		}
 
 		// Apply static IP if configured
@@ -112,8 +122,74 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine
 			},
 		}
 		machines = append(machines, machine)
+
+		vsphereMachineProvider = provider.DeepCopy()
+	}
+
+	// when multiple zones are defined, network and workspace are derived from the topology
+	origProv := vsphereMachineProvider.DeepCopy()
+	if len(failureDomains) > 1 {
+		vsphereMachineProvider.Network = machineapi.NetworkSpec{}
+		vsphereMachineProvider.Workspace = &machineapi.Workspace{}
+		vsphereMachineProvider.Template = ""
+	}
+
+	if len(hosts) > 0 {
+		vsphereMachineProvider.Network.Devices = []machineapi.NetworkDeviceSpec{
+			{
+				AddressesFromPools: origProv.Network.Devices[0].AddressesFromPools,
+				Nameservers:        origProv.Network.Devices[0].Nameservers,
+			},
+		}
+	}
+
+	controlPlaneMachineSet := &machinev1.ControlPlaneMachineSet{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "machine.openshift.io/v1",
+			Kind:       "ControlPlaneMachineSet",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "openshift-machine-api",
+			Name:      "cluster",
+			Labels: map[string]string{
+				"machine.openshift.io/cluster-api-cluster": clusterID,
+			},
+		},
+		Spec: machinev1.ControlPlaneMachineSetSpec{
+			Replicas: &replicas,
+			State:    machinev1.ControlPlaneMachineSetStateActive,
+			Selector: metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"machine.openshift.io/cluster-api-machine-role": role,
+					"machine.openshift.io/cluster-api-machine-type": role,
+					"machine.openshift.io/cluster-api-cluster":      clusterID,
+				},
+			},
+			Template: machinev1.ControlPlaneMachineSetTemplate{
+				MachineType: machinev1.OpenShiftMachineV1Beta1MachineType,
+				OpenShiftMachineV1Beta1Machine: &machinev1.OpenShiftMachineV1Beta1MachineTemplate{
+					FailureDomains: &machinev1.FailureDomains{
+						Platform: v1.VSpherePlatformType,
+						VSphere:  failureDomains,
+					},
+					ObjectMeta: machinev1.ControlPlaneMachineSetTemplateObjectMeta{
+						Labels: map[string]string{
+							"machine.openshift.io/cluster-api-cluster":      clusterID,
+							"machine.openshift.io/cluster-api-machine-role": role,
+							"machine.openshift.io/cluster-api-machine-type": role,
+						},
+					},
+					Spec: machineapi.MachineSpec{
+						ProviderSpec: machineapi.ProviderSpec{
+							Value: &runtime.RawExtension{Object: vsphereMachineProvider},
+						},
+					},
+				},
+			},
+		},
 	}
-	return machines, nil
+
+	return machines, controlPlaneMachineSet, nil
 }
 
 // applyNetworkConfig this function will apply the static ip configuration to the networkDevice

pkg/asset/machines/vsphere/machines_test.go

@@ -378,7 +378,7 @@ func TestConfigMasters(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.testCase, func(t *testing.T) {
-			machines, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", "", "")
+			machines, _, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", "", "")
 			assertOnUnexpectedErrorState(t, tc.expectedError, err)
 
 			if len(tc.workspaces) > 0 {
@@ -449,7 +449,7 @@ func TestHostsToMachines(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.testCase, func(t *testing.T) {
-			machines, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", tc.role, "")
+			machines, _, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", tc.role, "")
 			assertOnUnexpectedErrorState(t, tc.expectedError, err)
 
 			// Check machine counts

pkg/asset/machines/worker.go

@@ -627,7 +627,7 @@ func (w *Worker) Generate(dependencies asset.Parents) error {
 				logrus.Debug("Generating worker machines with static IPs.")
 				templateName := clusterID.InfraID + "-rhcos"
 
-				machines, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "worker", workerUserDataSecretName)
+				machines, _, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "worker", workerUserDataSecretName)
 				if err != nil {
 					return errors.Wrap(err, "failed to create worker machine objects")
 				}

pkg/asset/manifests/infrastructure.go

@@ -245,8 +245,7 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 			}
 		}
 
-		config.Spec.PlatformSpec.VSphere = vsphereinfra.GetInfraPlatformSpec(installConfig)
-
+		config.Spec.PlatformSpec.VSphere = vsphereinfra.GetInfraPlatformSpec(installConfig, clusterID.InfraID)
 		if _, exists := cloudproviderconfig.ConfigMap.Data["vsphere.conf"]; exists {
 			cloudProviderConfigMapKey = "vsphere.conf"
 		}

pkg/asset/manifests/vsphere/infrastructure.go

@@ -1,12 +1,14 @@
 package vsphere
 
 import (
+	"fmt"
+
 	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 )
 
 // GetInfraPlatformSpec constructs VSpherePlatformSpec for the infrastructure spec
-func GetInfraPlatformSpec(ic *installconfig.InstallConfig) *configv1.VSpherePlatformSpec {
+func GetInfraPlatformSpec(ic *installconfig.InstallConfig, clusterID string) *configv1.VSpherePlatformSpec {
 	var platformSpec configv1.VSpherePlatformSpec
 	icPlatformSpec := ic.Config.VSphere
 
@@ -21,6 +23,11 @@ func GetInfraPlatformSpec(ic *installconfig.InstallConfig) *configv1.VSpherePlat
 	for _, failureDomain := range icPlatformSpec.FailureDomains {
 		topology := failureDomain.Topology
 		if topology.ComputeCluster != "" && topology.Networks[0] != "" {
+			template := topology.Template
+			if len(template) == 0 {
+				template = fmt.Sprintf("/%s/vm/%s-rhcos-%s-%s", topology.Datacenter, clusterID, failureDomain.Region, failureDomain.Zone)
+			}
+
 			platformSpec.FailureDomains = append(platformSpec.FailureDomains, configv1.VSpherePlatformFailureDomainSpec{
 				Name:   failureDomain.Name,
 				Region: failureDomain.Region,
@@ -33,6 +40,7 @@ func GetInfraPlatformSpec(ic *installconfig.InstallConfig) *configv1.VSpherePlat
 					Datastore:      topology.Datastore,
 					ResourcePool:   topology.ResourcePool,
 					Folder:         topology.Folder,
+					Template:       template,
 				},
 			})
 		}