diff --git a/pkg/asset/cluster/tfvars/tfvars.go b/pkg/asset/cluster/tfvars/tfvars.go index ab9b2f82911..8f161c92b63 100644 --- a/pkg/asset/cluster/tfvars/tfvars.go +++ b/pkg/asset/cluster/tfvars/tfvars.go @@ -1035,6 +1035,10 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error { if err != nil { return err } + ipAddresses, err := mastersAsset.IPAddresses() + if err != nil { + return err + } controlPlaneConfigs := make([]*machinev1beta1.VSphereMachineProviderSpec, len(controlPlanes)) for i, c := range controlPlanes { var clusterMo mo.ClusterComputeResource @@ -1098,6 +1102,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error { InfraID: clusterID.InfraID, InstallConfig: installConfig, ControlPlaneMachines: controlPlanes, + IPAddresses: ipAddresses, }, ) if err != nil { diff --git a/pkg/asset/machines/master.go b/pkg/asset/machines/master.go index cf39bf1125d..09652b333ed 100644 --- a/pkg/asset/machines/master.go +++ b/pkg/asset/machines/master.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" "sigs.k8s.io/yaml" configv1 "github.com/openshift/api/config/v1" @@ -70,6 +71,8 @@ type Master struct { MachineConfigFiles []*asset.File MachineFiles []*asset.File ControlPlaneMachineSet *asset.File + IPClaimFiles []*asset.File + IPAddrFiles []*asset.File // SecretFiles is used by the baremetal platform to register the // credential information for communicating with management @@ -109,8 +112,14 @@ const ( // user-data secret. masterUserDataFileName = "99_openshift-cluster-api_master-user-data-secret.yaml" - // masterUserDataFileName is the filename used for the control plane machine sets. + // controlPlaneMachineSetFileName is the filename used for the control plane machine sets. controlPlaneMachineSetFileName = "99_openshift-machine-api_master-control-plane-machine-set.yaml" + + // ipClaimFileName is the filename used for the ip claims list. + ipClaimFileName = "99_openshift-machine-api_claim-%s.yaml" + + // ipAddressFileName is the filename used for the ip addresses list. + ipAddressFileName = "99_openshift-machine-api_address-%s.yaml" ) var ( @@ -118,6 +127,8 @@ var ( networkConfigSecretFileNamePattern = fmt.Sprintf(networkConfigSecretFileName, "*") hostFileNamePattern = fmt.Sprintf(hostFileName, "*") masterMachineFileNamePattern = fmt.Sprintf(masterMachineFileName, "*") + masterIPClaimFileNamePattern = fmt.Sprintf(ipClaimFileName, "*master*") + masterIPAddressFileNamePattern = fmt.Sprintf(ipAddressFileName, "*master*") _ asset.WritableAsset = (*Master)(nil) ) @@ -158,6 +169,8 @@ func (m *Master) Generate(dependencies asset.Parents) error { pool := *ic.ControlPlane var err error machines := []machinev1beta1.Machine{} + var ipClaims []ipamv1.IPAddressClaim + var ipAddrs []ipamv1.IPAddress var controlPlaneMachineSet *machinev1.ControlPlaneMachineSet switch ic.Platform.Name() { case awstypes.Name: @@ -439,10 +452,14 @@ func (m *Master) Generate(dependencies asset.Parents) error { pool.Platform.VSphere = &mpool templateName := clusterID.InfraID + "-rhcos" - machines, controlPlaneMachineSet, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName) + data, err := vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "master", masterUserDataSecretName) if err != nil { return errors.Wrap(err, "failed to create master machine objects") } + machines = data.Machines + controlPlaneMachineSet = data.ControlPlaneMachineSet + ipClaims = data.IPClaims + ipAddrs = data.IPAddresses if ic.FeatureSet != configv1.TechPreviewNoUpgrade { controlPlaneMachineSet = nil @@ -563,6 +580,33 @@ func (m *Master) Generate(dependencies asset.Parents) error { Data: data, } } + + m.IPClaimFiles = make([]*asset.File, len(ipClaims)) + for i, claim := range ipClaims { + data, err := yaml.Marshal(claim) + if err != nil { + return errors.Wrapf(err, "unable to marshal ip claim %v", claim.Name) + } + + m.IPClaimFiles[i] = &asset.File{ + Filename: filepath.Join(directory, fmt.Sprintf(ipClaimFileName, claim.Name)), + Data: data, + } + } + + m.IPAddrFiles = make([]*asset.File, len(ipAddrs)) + for i, address := range ipAddrs { + data, err := yaml.Marshal(address) + if err != nil { + return errors.Wrapf(err, "unable to marshal ip claim %v", address.Name) + } + + m.IPAddrFiles[i] = &asset.File{ + Filename: filepath.Join(directory, fmt.Sprintf(ipAddressFileName, address.Name)), + Data: data, + } + } + padFormat := fmt.Sprintf("%%0%dd", len(fmt.Sprintf("%d", len(machines)))) for i, machine := range machines { data, err := yaml.Marshal(machine) @@ -598,6 +642,8 @@ func (m *Master) Files() []*asset.File { if m.ControlPlaneMachineSet != nil { files = append(files, m.ControlPlaneMachineSet) } + files = append(files, m.IPClaimFiles...) + files = append(files, m.IPAddrFiles...) return files } @@ -654,6 +700,20 @@ func (m *Master) Load(f asset.FileFetcher) (found bool, err error) { } m.ControlPlaneMachineSet = file + fileList, err = f.FetchByPattern(filepath.Join(directory, masterIPClaimFileNamePattern)) + if err != nil { + return true, err + } + logrus.Infof("Loaded %v ip claims.", len(fileList)) + m.IPClaimFiles = fileList + + fileList, err = f.FetchByPattern(filepath.Join(directory, masterIPAddressFileNamePattern)) + if err != nil { + return true, err + } + logrus.Infof("Loaded %v ip addresses.", len(fileList)) + m.IPAddrFiles = fileList + return true, nil } @@ -735,6 +795,8 @@ func IsMachineManifest(file *asset.File) bool { {Pattern: hostFileNamePattern, Type: "host"}, {Pattern: masterMachineFileNamePattern, Type: "master machine"}, {Pattern: workerMachineSetFileNamePattern, Type: "worker machineset"}, + {Pattern: masterIPAddressFileNamePattern, Type: "master ip address"}, + {Pattern: masterIPClaimFileNamePattern, Type: "master ip address claim"}, } { if matched, err := filepath.Match(pattern.Pattern, filename); err != nil { panic(fmt.Sprintf("bad format for %s file name pattern", pattern.Type)) @@ -745,6 +807,23 @@ func IsMachineManifest(file *asset.File) bool { return false } +// IPAddresses returns IPAddress manifest structures. +func (m *Master) IPAddresses() ([]ipamv1.IPAddress, error) { + ipAddresses := []ipamv1.IPAddress{} + for i, file := range m.IPAddrFiles { + logrus.Debugf("Attempting to load address %v.", file.Filename) + address := &ipamv1.IPAddress{} + err := yaml.Unmarshal(file.Data, &address) + if err != nil { + return ipAddresses, errors.Wrapf(err, "unable to unmarshal ip address %d", i) + } + + ipAddresses = append(ipAddresses, *address) + } + + return ipAddresses, nil +} + func createSecretAssetFiles(resources []corev1.Secret, fileName string) ([]*asset.File, error) { var objects []interface{} diff --git a/pkg/asset/machines/vsphere/capimachines.go b/pkg/asset/machines/vsphere/capimachines.go index 368b225cbd3..d89ad11e967 100644 --- a/pkg/asset/machines/vsphere/capimachines.go +++ b/pkg/asset/machines/vsphere/capimachines.go @@ -21,6 +21,10 @@ import ( "github.com/openshift/installer/pkg/types" ) +const ( + masterRole = "master" +) + // ProviderSpecFromRawExtension unmarshals the JSON-encoded spec. func ProviderSpecFromRawExtension(rawExtension *runtime.RawExtension) (*machinev1.VSphereMachineProviderSpec, error) { if rawExtension == nil { @@ -57,10 +61,11 @@ func getNetworkInventoryPath(vcenterContext vsphere.VCenterContext, networkName // GenerateMachines returns a list of capi machines. func GenerateMachines(ctx context.Context, clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage string, role string, metadata *vsphere.Metadata) ([]*asset.RuntimeFile, error) { - machines, _, err := Machines(clusterID, config, pool, osImage, role, "") + data, err := Machines(clusterID, config, pool, osImage, role, "") if err != nil { return nil, fmt.Errorf("unable to retrieve machines: %w", err) } + machines := data.Machines capvMachines := make([]*capv.VSphereMachine, 0, len(machines)) result := make([]*asset.RuntimeFile, 0, len(machines)) @@ -157,8 +162,8 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta }) } - // as part of provisioning conrtol plane nodes, we need to create a bootstrap node as well - if role == "master" { + // as part of provisioning control plane nodes, we need to create a bootstrap node as well + if role == masterRole { customVMXKeys := map[string]string{} bootstrapSpec := capvMachines[0].Spec diff --git a/pkg/asset/machines/vsphere/machines.go b/pkg/asset/machines/vsphere/machines.go index bc8a6b8779a..89c205d4146 100644 --- a/pkg/asset/machines/vsphere/machines.go +++ b/pkg/asset/machines/vsphere/machines.go @@ -4,12 +4,15 @@ package vsphere import ( "fmt" "path" + "strconv" + "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" v1 "github.com/openshift/api/config/v1" machinev1 "github.com/openshift/api/machine/v1" @@ -18,17 +21,25 @@ import ( "github.com/openshift/installer/pkg/types/vsphere" ) +// MachineData contains all result output from the Machines() function. +type MachineData struct { + Machines []machineapi.Machine + ControlPlaneMachineSet *machinev1.ControlPlaneMachineSet + IPClaims []ipamv1.IPAddressClaim + IPAddresses []ipamv1.IPAddress +} + // Machines returns a list of machines for a machinepool. -func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string) ([]machineapi.Machine, *machinev1.ControlPlaneMachineSet, error) { +func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, osImage, role, userDataSecret string) (*MachineData, error) { + data := &MachineData{} if configPlatform := config.Platform.Name(); configPlatform != vsphere.Name { - return nil, nil, fmt.Errorf("non vsphere configuration: %q", configPlatform) + return data, fmt.Errorf("non vsphere configuration: %q", configPlatform) } if poolPlatform := pool.Platform.Name(); poolPlatform != vsphere.Name { - return nil, nil, fmt.Errorf("non-VSphere machine-pool: %q", poolPlatform) + return data, fmt.Errorf("non-VSphere machine-pool: %q", poolPlatform) } var failureDomain vsphere.FailureDomain - var machines []machineapi.Machine platform := config.Platform.VSphere mpool := pool.Platform.VSphere replicas := int32(1) @@ -37,7 +48,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine zones, err := getDefinedZonesFromTopology(platform) if err != nil { - return machines, nil, err + return data, err } if pool.Replicas != nil { @@ -73,7 +84,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine logrus.Debugf("Desired zone: %v", desiredZone) if _, exists := zones[desiredZone]; !exists { - return nil, nil, errors.Errorf("zone [%s] specified by machinepool is not defined", desiredZone) + return data, errors.Errorf("zone [%s] specified by machinepool is not defined", desiredZone) } failureDomain = zones[desiredZone] @@ -97,16 +108,13 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine vcenter, err := getVCenterFromServerName(failureDomain.Server, platform) if err != nil { - return nil, nil, errors.Wrap(err, "unable to find vCenter in failure domains") + return data, errors.Wrap(err, "unable to find vCenter in failure domains") } provider, err := provider(clusterID, vcenter, failureDomain, mpool, osImageForZone, userDataSecret) if err != nil { - return nil, nil, errors.Wrap(err, "failed to create provider") + return data, errors.Wrap(err, "failed to create provider") } - // Apply static IP if configured - applyNetworkConfig(host, provider) - machine := machineapi.Machine{ TypeMeta: metav1.TypeMeta{ APIVersion: "machine.openshift.io/v1beta1", @@ -124,7 +132,16 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine // we don't need to set Versions, because we control those via operators. }, } - machines = append(machines, machine) + + // Apply static IP if configured + claim, address, err := applyNetworkConfig(host, provider, machine) + if err != nil { + return data, err + } else if claim != nil && address != nil { + data.IPClaims = append(data.IPClaims, claim...) + data.IPAddresses = append(data.IPAddresses, address...) + } + data.Machines = append(data.Machines, machine) vsphereMachineProvider = provider.DeepCopy() } @@ -153,7 +170,7 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine } } - controlPlaneMachineSet := &machinev1.ControlPlaneMachineSet{ + data.ControlPlaneMachineSet = &machinev1.ControlPlaneMachineSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "machine.openshift.io/v1", Kind: "ControlPlaneMachineSet", @@ -199,21 +216,100 @@ func Machines(clusterID string, config *types.InstallConfig, pool *types.Machine }, } - return machines, controlPlaneMachineSet, nil + return data, nil } // applyNetworkConfig this function will apply the static ip configuration to the networkDevice // field in the provider spec. The function will use the desired zone to determine which config // to apply and then remove that host config from the hosts array. -func applyNetworkConfig(host *vsphere.Host, provider *machineapi.VSphereMachineProviderSpec) { +func applyNetworkConfig(host *vsphere.Host, provider *machineapi.VSphereMachineProviderSpec, machine machineapi.Machine) ([]ipamv1.IPAddressClaim, []ipamv1.IPAddress, error) { + var ipClaims []ipamv1.IPAddressClaim + var ipAddrs []ipamv1.IPAddress if host != nil { networkDevice := host.NetworkDevice if networkDevice != nil { - provider.Network.Devices[0].IPAddrs = networkDevice.IPAddrs - provider.Network.Devices[0].Nameservers = networkDevice.Nameservers - provider.Network.Devices[0].Gateway = networkDevice.Gateway + for idx, address := range networkDevice.IPAddrs { + provider.Network.Devices[0].Nameservers = networkDevice.Nameservers + provider.Network.Devices[0].AddressesFromPools = append(provider.Network.Devices[0].AddressesFromPools, machineapi.AddressesFromPool{ + Group: "installer.openshift.io", + Name: fmt.Sprintf("default-%d", idx), + Resource: "IPPool", + }, + ) + + // Generate the capi networking objects + slashIndex := strings.Index(address, "/") + ipAddress := address[0:slashIndex] + prefix, err := strconv.Atoi(address[slashIndex+1:]) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to determine address prefix") + } + ipClaim, ipAddr := generateCapiNetwork(machine.Name, ipAddress, networkDevice.Gateway, prefix, 0, idx) + ipClaims = append(ipClaims, *ipClaim) + ipAddrs = append(ipAddrs, *ipAddr) + } } } + + return ipClaims, ipAddrs, nil +} + +// generateCapiNetwork this function will create IPAddressClaim and IPAddress for the specified information. +func generateCapiNetwork(machineName, ipAddress, gateway string, prefix, deviceIndex, ipIndex int) (*ipamv1.IPAddressClaim, *ipamv1.IPAddress) { + // Generate PoolRef + apigroup := "installer.openshift.io" + poolRef := corev1.TypedLocalObjectReference{ + APIGroup: &apigroup, + Kind: "IPPool", + Name: fmt.Sprintf("default-%d", ipIndex), + } + + // Generate IPAddressClaim + ipclaim := &ipamv1.IPAddressClaim{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "ipam.cluster.x-k8s.io/v1beta1", + Kind: "IPAddressClaim", + }, + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{ + machineapi.IPClaimProtectionFinalizer, + }, + Name: fmt.Sprintf("%s-claim-%d-%d", machineName, deviceIndex, ipIndex), + Namespace: "openshift-machine-api", + }, + Spec: ipamv1.IPAddressClaimSpec{ + PoolRef: poolRef, + }, + } + + // Populate IPAddress info + ipaddr := &ipamv1.IPAddress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "ipam.cluster.x-k8s.io/v1beta1", + Kind: "IPAddress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-claim-%d-%d", machineName, deviceIndex, ipIndex), + Namespace: "openshift-machine-api", + }, + Spec: ipamv1.IPAddressSpec{ + Address: ipAddress, + ClaimRef: corev1.LocalObjectReference{ + Name: ipclaim.Name, + }, + Gateway: gateway, + PoolRef: poolRef, + Prefix: prefix, + }, + } + + ipclaim.Status = ipamv1.IPAddressClaimStatus{ + AddressRef: corev1.LocalObjectReference{ + Name: ipaddr.Name, + }, + } + + return ipclaim, ipaddr } func provider(clusterID string, vcenter *vsphere.VCenter, failureDomain vsphere.FailureDomain, mpool *vsphere.MachinePool, osImage string, userDataSecret string) (*machineapi.VSphereMachineProviderSpec, error) { diff --git a/pkg/asset/machines/vsphere/machines_test.go b/pkg/asset/machines/vsphere/machines_test.go index 39ba57610f4..b49e8707e85 100644 --- a/pkg/asset/machines/vsphere/machines_test.go +++ b/pkg/asset/machines/vsphere/machines_test.go @@ -413,7 +413,7 @@ func TestConfigMasters(t *testing.T) { for _, tc := range testCases { t.Run(tc.testCase, func(t *testing.T) { - machines, cpms, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", "", "") + data, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", "", "") assertOnUnexpectedErrorState(t, tc.expectedError, err) if len(tc.workspaces) > 0 { @@ -422,7 +422,7 @@ func TestConfigMasters(t *testing.T) { matchCountByIndex = append(matchCountByIndex, 0) } - for _, machine := range machines { + for _, machine := range data.Machines { // check if expected workspaces are returned machineWorkspace := machine.Spec.ProviderSpec.Value.Object.(*machineapi.VSphereMachineProviderSpec).Workspace for idx, workspace := range tc.workspaces { @@ -440,9 +440,9 @@ func TestConfigMasters(t *testing.T) { } } - if cpms != nil { + if data.ControlPlaneMachineSet != nil { // Make sure FDs equal same quantity as config - fds := cpms.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains + fds := data.ControlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.FailureDomains if len(fds.VSphere) != len(tc.workspaces) { t.Errorf("machine workspace count %d does not equal number of failure domains [count: %d] in CPMS", len(tc.workspaces), len(fds.VSphere)) } @@ -492,16 +492,26 @@ func TestHostsToMachines(t *testing.T) { for _, tc := range testCases { t.Run(tc.testCase, func(t *testing.T) { - machines, _, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", tc.role, "") + data, err := Machines(clusterID, tc.installConfig, tc.machinePool, "", tc.role, "") assertOnUnexpectedErrorState(t, tc.expectedError, err) // Check machine counts - if len(machines) != tc.machineCount { - t.Errorf("machine count (%v) did not match expected (%v).", len(machines), tc.machineCount) + if len(data.Machines) != tc.machineCount { + t.Errorf("machine count (%v) did not match expected (%v).", len(data.Machines), tc.machineCount) + } + + // Check Claim counts + if len(data.IPClaims) != tc.machineCount { + t.Errorf("ip address claim count (%v) did not match expected (%v).", len(data.IPClaims), tc.machineCount) + } + + // Check ip address counts + if len(data.IPAddresses) != tc.machineCount { + t.Errorf("ip address count (%v) did not match expected (%v).", len(data.IPAddresses), tc.machineCount) } // Verify static IP was set on all machines - for _, machine := range machines { + for index, machine := range data.Machines { provider, success := machine.Spec.ProviderSpec.Value.Object.(*machineapi.VSphereMachineProviderSpec) if !success { t.Errorf("Unable to convert vshere machine provider spec.") @@ -509,8 +519,8 @@ func TestHostsToMachines(t *testing.T) { if len(provider.Network.Devices) == 1 { // Check IP - if provider.Network.Devices[0].IPAddrs == nil || provider.Network.Devices[0].IPAddrs[0] == "" { - t.Errorf("Static ip is not set: %v", machine) + if provider.Network.Devices[0].AddressesFromPools == nil { + t.Errorf("AddressesFromPools is not set: %v", machine) } // Check nameserver @@ -518,7 +528,7 @@ func TestHostsToMachines(t *testing.T) { t.Errorf("Nameserver is not set: %v", machine) } - gateway := provider.Network.Devices[0].Gateway + gateway := data.IPAddresses[index].Spec.Gateway ip, err := netip.ParseAddr(gateway) if err != nil { t.Error(err) diff --git a/pkg/asset/machines/vsphere/machinesets.go b/pkg/asset/machines/vsphere/machinesets.go index 1ef3dbd0a95..e376ae444ce 100644 --- a/pkg/asset/machines/vsphere/machinesets.go +++ b/pkg/asset/machines/vsphere/machinesets.go @@ -22,12 +22,37 @@ func getMachineSetWithPlatform( vcenter *vsphere.VCenter, replicas int32, role, - userDataSecret string) (*machineapi.MachineSet, error) { + userDataSecret string, + hosts []*vsphere.Host) (*machineapi.MachineSet, error) { provider, err := provider(clusterID, vcenter, failureDomain, mpool, osImage, userDataSecret) if err != nil { return nil, errors.Wrap(err, "failed to create provider") } + // If static IP, create the addressesFromPools + if len(hosts) > 0 { + // Get an example host def for network specific nameserver + var nameservers []string + for _, host := range hosts { + if (host.IsCompute() && role == "worker") || (host.IsControlPlane() && role == "master") { + nameservers = host.NetworkDevice.Nameservers + break + } + } + + // Generate AddressesFromPools for each device + for idx := range provider.Network.Devices { + provider.Network.Devices[idx].Nameservers = nameservers + provider.Network.Devices[idx].AddressesFromPools = []machineapi.AddressesFromPool{ + { + Group: "installer.openshift.io", + Name: fmt.Sprintf("default-%d", idx), + Resource: "IPPool", + }, + } + } + } + mset := &machineapi.MachineSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "machine.openshift.io/v1beta1", @@ -148,7 +173,8 @@ func MachineSets(clusterID string, config *types.InstallConfig, pool *types.Mach vcenter, replicas, role, - userDataSecret) + userDataSecret, + platform.Hosts) if err != nil { return machinesets, err } diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index b72afc40bcf..b3b197037a5 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" "sigs.k8s.io/yaml" configv1 "github.com/openshift/api/config/v1" @@ -90,6 +91,8 @@ const ( var ( workerMachineSetFileNamePattern = fmt.Sprintf(workerMachineSetFileName, "*") workerMachineFileNamePattern = fmt.Sprintf(workerMachineFileName, "*") + workerIPClaimFileNamePattern = fmt.Sprintf(ipClaimFileName, "*worker*") + workerIPAddressFileNamePattern = fmt.Sprintf(ipAddressFileName, "*worker*") _ asset.WritableAsset = (*Worker)(nil) ) @@ -239,6 +242,8 @@ type Worker struct { MachineConfigFiles []*asset.File MachineSetFiles []*asset.File MachineFiles []*asset.File + IPClaimFiles []*asset.File + IPAddrFiles []*asset.File } // Name returns a human friendly name for the Worker Asset. @@ -277,6 +282,8 @@ func (w *Worker) Generate(dependencies asset.Parents) error { machines := []machinev1beta1.Machine{} machineConfigs := []*mcfgv1.MachineConfig{} machineSets := []runtime.Object{} + var ipClaims []ipamv1.IPAddressClaim + var ipAddrs []ipamv1.IPAddress var err error ic := installConfig.Config for _, pool := range ic.Compute { @@ -604,10 +611,15 @@ func (w *Worker) Generate(dependencies asset.Parents) error { logrus.Debug("Generating worker machines with static IPs.") templateName := clusterID.InfraID + "-rhcos" - machines, _, err = vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "worker", workerUserDataSecretName) + data, err := vsphere.Machines(clusterID.InfraID, ic, &pool, templateName, "worker", workerUserDataSecretName) if err != nil { return errors.Wrap(err, "failed to create worker machine objects") } + + machines = data.Machines + ipClaims = data.IPClaims + ipAddrs = data.IPAddresses + logrus.Debugf("Generated %v worker machines.", len(machines)) for _, ms := range sets { @@ -693,6 +705,31 @@ func (w *Worker) Generate(dependencies asset.Parents) error { } } + w.IPClaimFiles = make([]*asset.File, len(ipClaims)) + for i, claim := range ipClaims { + data, err := yaml.Marshal(claim) + if err != nil { + return errors.Wrapf(err, "marshal ip claim %v", claim.Name) + } + + w.IPClaimFiles[i] = &asset.File{ + Filename: filepath.Join(directory, fmt.Sprintf(ipClaimFileName, claim.Name)), + Data: data, + } + } + + w.IPAddrFiles = make([]*asset.File, len(ipAddrs)) + for i, address := range ipAddrs { + data, err := yaml.Marshal(address) + if err != nil { + return errors.Wrapf(err, "marshal ip claim %v", address.Name) + } + + w.IPAddrFiles[i] = &asset.File{ + Filename: filepath.Join(directory, fmt.Sprintf(ipAddressFileName, address.Name)), + Data: data, + } + } w.MachineFiles = make([]*asset.File, len(machines)) for i, machineDef := range machines { data, err := yaml.Marshal(machineDef) @@ -718,6 +755,8 @@ func (w *Worker) Files() []*asset.File { files = append(files, w.MachineConfigFiles...) files = append(files, w.MachineSetFiles...) files = append(files, w.MachineFiles...) + files = append(files, w.IPClaimFiles...) + files = append(files, w.IPAddrFiles...) return files } @@ -750,6 +789,18 @@ func (w *Worker) Load(f asset.FileFetcher) (found bool, err error) { } w.MachineFiles = fileList + fileList, err = f.FetchByPattern(filepath.Join(directory, workerIPClaimFileNamePattern)) + if err != nil { + return true, err + } + w.IPClaimFiles = fileList + + fileList, err = f.FetchByPattern(filepath.Join(directory, workerIPAddressFileNamePattern)) + if err != nil { + return true, err + } + w.IPAddrFiles = fileList + return true, nil } diff --git a/pkg/tfvars/vsphere/vsphere.go b/pkg/tfvars/vsphere/vsphere.go index 617b5ed05c1..c8057f66bc4 100644 --- a/pkg/tfvars/vsphere/vsphere.go +++ b/pkg/tfvars/vsphere/vsphere.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1alpha1" + ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" machineapi "github.com/openshift/api/machine/v1beta1" "github.com/openshift/installer/pkg/asset/installconfig" @@ -169,6 +169,7 @@ func processGuestNetworkConfiguration(cfg *config, sources TFVarsSources) error } } + // Generate control plane kargs using info from machine network config // Current logic assumes only 1 network defined per machine for index, machine := range sources.ControlPlaneMachines { logrus.Infof("Generating kargs for control plane %v.", machine.Name)