From c2e46879bb54d033db33ca8ee65147f21b2f8037 Mon Sep 17 00:00:00 2001 From: Ori Amizur Date: Thu, 2 May 2024 14:29:07 +0300 Subject: [PATCH] MGMT-17700: Assign none platform node-ips based on connected addresses and etcd restrictions When none platform is in use, if there is ambiguity in node-ip assignment, then incorrect assignment might lead to installation failure. This happens when etcd detects that the socket address from an etcd node does not match the expected address in the peer certificate. In this case etcd rejects such connection. Example: assuming two networks - net1 and net2. master node 1 has 1 address that belongs to net1. master node 2 has 2 addresses. one that belongs to net 1, and another that belongs to net 2 master node 3 has 1 address that belongs to net 1. If the selected node-ip of master node 2 belongs to net 2, then when it will create a connection with any other master node, the socket address will be the address that belongs to net 1. Since etcd expects it to be the same as the node-ip, it will reject the connection. This can be solved by node-ips selection that will not cause such a conflict. Node ips assignment should be done through ignition. To correctly set bootstrap ip, the machine-network for the cluster must be set to match the selected node-ip for that host. MGMT-17701: Add capability to calculate none platform node-ips based on L3 connected addresses and connectivity Calculate the node-ips for none platform cluster. Node ip calculation is actually calculation of the node-ip, hint, and cidr. Node-ip is the ip address that exists on the host that was selected as the node-ip. Hint is actually an IP address that does not exist on the host, but must belong to the subnet of the node-ip. Cidr is the subnet in cidr notation that the node-ip and hint belong to. Node-ip calculation is either done for all cluster hosts or none of them. MGMT-17702: Modify ignition to use calculated node-ips for none platform In order to set node-ip, ignition is modified. A file called /etc/default/nodeip-configuration contains the hint as was set by node-ip generation. In addition the bootstrap-ip is set in ignition as was set in node-ip generation. This is set as environment variable called "OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP". MGMT-17703: Modify machine-network based on calculated node-ips The etcd in boostrap uses he machine-network to set the IP address. Therefore during install-config generation the machine-network may be replaced by the cidr from the node-ip generation. --- internal/common/common.go | 4 + internal/ignition/installmanifests.go | 38 +- internal/ignition/installmanifests_test.go | 77 +++ .../none_platform_node_ips_allocation.go | 321 ++++++++++++ .../none_platform_node_ips_allocation_test.go | 486 ++++++++++++++++++ internal/provider/none/installConfig.go | 32 ++ 6 files changed, 955 insertions(+), 3 deletions(-) create mode 100644 internal/network/none_platform_node_ips_allocation.go create mode 100644 internal/network/none_platform_node_ips_allocation_test.go diff --git a/internal/common/common.go b/internal/common/common.go index 044308e2320f..8ad08b18cae5 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -609,3 +609,7 @@ func IsExternalIntegrationEnabled(platform *models.Platform, platformName string func IsOciExternalIntegrationEnabled(platform *models.Platform) bool { return IsExternalIntegrationEnabled(platform, ExternalPlatformNameOci) } + +func IsMultiNodeNonePlatformCluster(cluster *Cluster) bool { + return !IsSingleNodeCluster(cluster) && swag.BoolValue(cluster.UserManagedNetworking) +} diff --git a/internal/ignition/installmanifests.go b/internal/ignition/installmanifests.go index 3091e4c5f112..357d47d82fb2 100644 --- a/internal/ignition/installmanifests.go +++ b/internal/ignition/installmanifests.go @@ -35,6 +35,7 @@ import ( "github.com/openshift/assisted-service/pkg/mirrorregistries" "github.com/openshift/assisted-service/pkg/s3wrapper" "github.com/pkg/errors" + "github.com/samber/lo" "github.com/sirupsen/logrus" "github.com/thoas/go-funk" "github.com/vincent-petithory/dataurl" @@ -92,6 +93,7 @@ type installerGenerator struct { installerReleaseImageOverride string clusterTLSCertOverrideDir string installerCache *installercache.Installers + nodeIpAllocations map[strfmt.UUID]*network.NodeIpAllocation } var fileNames = [...]string{ @@ -132,6 +134,17 @@ func (g *installerGenerator) UploadToS3(ctx context.Context) error { return uploadToS3(ctx, g.workDir, g.cluster, g.s3Client, g.log) } +func (g *installerGenerator) allocateNodeIpsIfNeeded(log logrus.FieldLogger) { + if common.IsMultiNodeNonePlatformCluster(g.cluster) { + nodeIpAllocations, err := network.GenerateNonePlatformAddressAllocation(g.cluster, log) + if err != nil { + log.WithError(err).Warnf("failed to generate ip address allocation for cluster %s", *g.cluster.ID) + } else { + g.nodeIpAllocations = nodeIpAllocations + } + } +} + // Generate generates ignition files and applies modifications. func (g *installerGenerator) Generate(ctx context.Context, installConfig []byte) error { var err error @@ -180,6 +193,8 @@ func (g *installerGenerator) Generate(ctx context.Context, installConfig []byte) return wrapped } + g.allocateNodeIpsIfNeeded(log) + envVars := append(os.Environ(), "OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="+g.releaseImage, "OPENSHIFT_INSTALL_INVOKER="+g.installInvoker, @@ -313,16 +328,27 @@ func (g *installerGenerator) Generate(ctx context.Context, installConfig []byte) func (g *installerGenerator) addBootstrapKubeletIpIfRequired(log logrus.FieldLogger, envVars []string) ([]string, error) { // setting bootstrap kubelet node ip - // We don't want to set bootstrap ip in None platform as user can't set machine cidr - //and we can choose the wrong one log.Debugf("Adding bootstrap ip to env vars") - if !swag.BoolValue(g.cluster.UserManagedNetworking) || common.IsSingleNodeCluster(g.cluster) { + if !common.IsMultiNodeNonePlatformCluster(g.cluster) { bootstrapIp, err := network.GetPrimaryMachineCIDRIP(common.GetBootstrapHost(g.cluster), g.cluster) if err != nil { log.WithError(err).Warn("Failed to get bootstrap primary ip for kubelet service update.") return envVars, err } envVars = append(envVars, "OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP="+bootstrapIp) + } else if g.nodeIpAllocations != nil { + bootstrapHost := common.GetBootstrapHost(g.cluster) + if bootstrapHost != nil { + allocation, ok := g.nodeIpAllocations[lo.FromPtr(bootstrapHost.ID)] + if ok { + envVars = append(envVars, "OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP="+allocation.NodeIp) + g.log.Infof("Set OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP=%s for host %s", allocation.NodeIp, lo.FromPtr(bootstrapHost.ID)) + + } + } + } else { + log.Errorf("Unable to set OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP for multinode none platform cluster %s - missing node ip allocations", + lo.FromPtr(g.cluster.ID)) } return envVars, nil } @@ -1047,6 +1073,12 @@ func (g *installerGenerator) writeSingleHostFile(host *models.Host, baseFile str return errors.Wrapf(errP, "Failed to parse machine cidr for node ip hint content") } setFileInIgnition(config, nodeIpHintFile, fmt.Sprintf("data:,KUBELET_NODEIP_HINT=%s", ip), false, 420, true) + } else if g.nodeIpAllocations != nil && common.IsMultiNodeNonePlatformCluster(g.cluster) { + allocation, ok := g.nodeIpAllocations[lo.FromPtr(host.ID)] + if ok { + setFileInIgnition(config, nodeIpHintFile, fmt.Sprintf("data:,KUBELET_NODEIP_HINT=%s", allocation.HintIp), false, 420, true) + g.log.Infof("Set KUBELET_NODEIP_HINT=%s for host %s", allocation.HintIp, lo.FromPtr(host.ID)) + } } configBytes, err := json.Marshal(config) diff --git a/internal/ignition/installmanifests_test.go b/internal/ignition/installmanifests_test.go index e5bd0f04d83d..328b56832d6a 100644 --- a/internal/ignition/installmanifests_test.go +++ b/internal/ignition/installmanifests_test.go @@ -598,6 +598,62 @@ var _ = Describe("createHostIgnitions", func() { }) }) + It("none platform: expect hint to be created when node-ip allocations exist", func() { + cluster.Hosts = []*models.Host{ + { + RequestedHostname: "master0.example.com", + Role: models.HostRoleMaster, + }, + { + RequestedHostname: "master1.example.com", + Role: models.HostRoleMaster, + }, + { + RequestedHostname: "master2.example.com", + Role: models.HostRoleMaster, + }, + } + cluster.HighAvailabilityMode = swag.String(models.ClusterHighAvailabilityModeFull) + cluster.UserManagedNetworking = swag.Bool(true) + + // create an ID for each host + for _, host := range cluster.Hosts { + id := strfmt.UUID(uuid.New().String()) + host.ID = &id + } + + g := NewGenerator(workDir, "", cluster, "", "", "", "", nil, logrus.New(), nil, "", "", 5).(*installerGenerator) + g.nodeIpAllocations = make(map[strfmt.UUID]*network.NodeIpAllocation) + for i, h := range cluster.Hosts { + g.nodeIpAllocations[*h.ID] = &network.NodeIpAllocation{ + NodeIp: fmt.Sprintf("3.3.3.%d", i+1), + HintIp: "3.3.3.0", + Cidr: "3.3.3.0/24", + } + } + err := g.createHostIgnitions() + Expect(err).NotTo(HaveOccurred()) + + for _, host := range cluster.Hosts { + ignBytes, err := os.ReadFile(filepath.Join(workDir, fmt.Sprintf("%s-%s.ign", host.Role, host.ID))) + Expect(err).NotTo(HaveOccurred()) + config, _, err := config_32.Parse(ignBytes) + Expect(err).NotTo(HaveOccurred()) + + By("Validating nodeip hint was not added") + var f *config_32_types.File + for fileidx, file := range config.Storage.Files { + if file.Node.Path == nodeIpHintFile { + f = &config.Storage.Files[fileidx] + Expect(*f.Node.User.Name).To(Equal("root")) + Expect(*f.FileEmbedded1.Contents.Source).To(Equal(fmt.Sprintf("data:,KUBELET_NODEIP_HINT=%s", "3.3.3.0"))) + Expect(*f.FileEmbedded1.Mode).To(Equal(420)) + Expect(*f.Node.Overwrite).To(Equal(true)) + } + } + } + }) + It("applies overrides correctly", func() { hostID := strfmt.UUID(uuid.New().String()) cluster.Hosts = []*models.Host{{ @@ -1336,6 +1392,11 @@ var _ = Describe("Set kubelet node ip", func() { workDir string ) + uuidPtr := func() *strfmt.UUID { + ret := strfmt.UUID(uuid.New().String()) + return &ret + } + BeforeEach(func() { bootstrapInventory := `{"bmc_address":"0.0.0.0","bmc_v6address":"::/0","boot":{"current_boot_mode":"bios"},"cpu":{"architecture":"x86_64","count":4,"flags":["fpu","vme","de","pse","tsc","msr","pae","mce","cx8","apic","sep","mtrr","pge","mca","cmov","pat","pse36","clflush","mmx","fxsr","sse","sse2","ss","syscall","nx","pdpe1gb","rdtscp","lm","constant_tsc","arch_perfmon","rep_good","nopl","xtopology","cpuid","tsc_known_freq","pni","pclmulqdq","vmx","ssse3","fma","cx16","pcid","sse4_1","sse4_2","x2apic","movbe","popcnt","tsc_deadline_timer","aes","xsave","avx","f16c","rdrand","hypervisor","lahf_lm","abm","3dnowprefetch","cpuid_fault","invpcid_single","pti","ssbd","ibrs","ibpb","stibp","tpr_shadow","vnmi","flexpriority","ept","vpid","ept_ad","fsgsbase","tsc_adjust","bmi1","hle","avx2","smep","bmi2","erms","invpcid","rtm","mpx","avx512f","avx512dq","rdseed","adx","smap","clflushopt","clwb","avx512cd","avx512bw","avx512vl","xsaveopt","xsavec","xgetbv1","xsaves","arat","umip","pku","ospke","md_clear","arch_capabilities"],"frequency":2095.076,"model_name":"Intel(R) Xeon(R) Gold 6152 CPU @ 2.10GHz"},"disks":[{"by_path":"/dev/disk/by-path/pci-0000:00:06.0","drive_type":"HDD","model":"unknown","name":"vda","path":"/dev/vda","serial":"unknown","size_bytes":21474836480,"vendor":"0x1af4","wwn":"unknown"}],"hostname":"test-infra-cluster-master-1.redhat.com","interfaces":[{"flags":["up","broadcast","multicast"],"has_carrier":true,"ipv4_addresses":["192.168.126.10/24"],"ipv6_addresses":["fe80::5054:ff:fe42:1e8d/64"],"mac_address":"52:54:00:42:1e:8d","mtu":1500,"name":"eth0","product":"0x0001","speed_mbps":-1,"vendor":"0x1af4"},{"flags":["up","broadcast","multicast"],"has_carrier":true,"ipv4_addresses":["192.168.140.133/24"],"ipv6_addresses":["fe80::5054:ff:feca:7b16/64"],"mac_address":"52:54:00:ca:7b:16","mtu":1500,"name":"eth1","product":"0x0001","speed_mbps":-1,"vendor":"0x1af4"}],"memory":{"physical_bytes":17809014784,"usable_bytes":17378611200},"system_vendor":{"manufacturer":"Red Hat","product_name":"KVM"}}` @@ -1343,12 +1404,14 @@ var _ = Describe("Set kubelet node ip", func() { cluster.MachineNetworks = []*models.MachineNetwork{{Cidr: "192.168.126.0/24"}, {Cidr: "192.168.140.0/24"}} cluster.Hosts = []*models.Host{ { + ID: uuidPtr(), Inventory: bootstrapInventory, RequestedHostname: "example0", Role: models.HostRoleMaster, Bootstrap: true, }, { + ID: uuidPtr(), Inventory: hostInventory, RequestedHostname: "example1", Role: models.HostRoleMaster, @@ -1393,6 +1456,20 @@ var _ = Describe("Set kubelet node ip", func() { Expect(basicEnvVars).ShouldNot(ContainElement("OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP=192.168.126.10")) Expect(len(basicEnvVars)).Should(Equal(1)) }) + It("UMN platform bootstrap kubelet ip should be set when node-ip-allocations exist", func() { + cluster.UserManagedNetworking = swag.Bool(true) + cluster.HighAvailabilityMode = swag.String(models.ClusterHighAvailabilityModeFull) + generator.nodeIpAllocations = map[strfmt.UUID]*network.NodeIpAllocation{ + *cluster.Hosts[0].ID: { + NodeIp: "192.168.126.11", + HintIp: "192.168.126.0", + Cidr: "192.168.126.0/24", + }, + } + basicEnvVars, err = generator.addBootstrapKubeletIpIfRequired(generator.log, basicEnvVars) + Expect(err).NotTo(HaveOccurred()) + Expect(basicEnvVars).Should(ContainElement("OPENSHIFT_INSTALL_BOOTSTRAP_NODE_IP=192.168.126.11")) + }) It("should fail if no machine networks exists", func() { cluster.MachineNetworks = []*models.MachineNetwork{} basicEnvVars, err = generator.addBootstrapKubeletIpIfRequired(generator.log, basicEnvVars) diff --git a/internal/network/none_platform_node_ips_allocation.go b/internal/network/none_platform_node_ips_allocation.go new file mode 100644 index 000000000000..9aa35ec4903b --- /dev/null +++ b/internal/network/none_platform_node_ips_allocation.go @@ -0,0 +1,321 @@ +package network + +import ( + "encoding/json" + stderrors "errors" + "fmt" + "net" + "sort" + "strings" + + "github.com/go-openapi/strfmt" + "github.com/openshift/assisted-service/internal/common" + "github.com/openshift/assisted-service/models" + "github.com/pkg/errors" + "github.com/samber/lo" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +// When none platform is in use, if there is ambiguity in node-ip assignment, then incorrect assignment might lead +// to installation failure. This happens when etcd detects that the socket address from an etcd node does not match +// the expected address in the peer certificate. In this case etcd rejects such connection. +// Example: assuming two networks - net1 and net2. +// master node 1 has 1 address that belongs to net1. +// master node 2 has 2 addresses. one that belongs to net 1, and another that belongs to net 2 +// master node 3 has 1 address that belongs to net 1. +// If the selected node-ip of master node 2 belongs to net 2, then when it will create a connection with any other master node, +// the socket address will be the address that belongs to net 1. Since etcd expects it to be the same as the node-ip, it will +// reject the connection. +// A similar situation may happen with tunnel IPs (Geneve for OVN). When setting tunnel, OVS configuration contains the +// node-ips as endpoints. When there is a collision, the receiving endpoint of a tunnel may receive a tunnel packet +// from an IP address it does expect. This leads OVS to ignore the packet. Such situation may occur for master and worker +// nodes. If such collision happens for workers, there is a possibility that the cluster will still be functional. +// +// This file attempts to solve the issue automatically. + +// NodeIpAllocation is the allocation result for specific host +type NodeIpAllocation struct { + NodeIp string + HintIp string + Cidr string +} + +// nodeIpCandidate is an assignment candidate which is used when attempting to find optimal solution +type nodeIpCandidate struct { + hostId strfmt.UUID + ip net.IP +} + +// hostNetwork represents discovered network in a host +type hostNetwork struct { + hostId strfmt.UUID + ipNet *net.IPNet +} + +// eligibleNodeIp is an ip with its network which eligible for allocation. It means the IP is connected to all other hosts +// and its interface has default route +type eligibleNodeIp struct { + nodeIp net.IP + ipNet *net.IPNet + routeMetric int32 +} + +// hostCandidates gathers all eligible node IPs and all the discovered networks in that host +type hostCandidates struct { + hostId strfmt.UUID + hostNetworks []*hostNetwork + eligibleNodeIps []*eligibleNodeIp +} + +// allocationSolution contains all or partial set of the allocation. +type allocationSolution struct { + routeMetric int32 + nodeIpAllocations map[strfmt.UUID]*NodeIpAllocation +} + +func getDefaultMetricForInterface(interfaceName string, routes []*models.Route, isIpv4 bool) (bool, int32) { + for _, r := range routes { + if isIpv4 && r.Family != unix.AF_INET || !isIpv4 && r.Family != unix.AF_INET6 { + continue + } + isDefault, err := IsDefaultRoute(r) + if err != nil { + continue + } + if isDefault && r.Interface == interfaceName { + return true, r.Metric + } + } + return false, 0 +} + +// Create candidates for host. For an address to be considered as eligible IP, it has to be connected, and its interface must +// have default route. +func createHostCandidates(host *models.Host, connectivity *Connectivity, isIpv4 bool) (*hostCandidates, error) { + connectedAddresses := connectivity.L3ConnectedAddresses[lo.FromPtr(host.ID)] + if len(connectedAddresses) == 0 { + return nil, errors.Errorf("no address found for host %s", lo.FromPtr(host.ID)) + } + connectedIps := lo.Map(lo.Filter(connectedAddresses, func(addrStr string, _ int) bool { return isIpv4 == IsIPv4Addr(addrStr) }), + func(addrStr string, _ int) net.IP { return net.ParseIP(addrStr) }) + inventory, err := common.UnmarshalInventory(host.Inventory) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal inventory for host %s", lo.FromPtr(host.ID)) + } + var ( + eligibleNodeIps []*eligibleNodeIp + hostNetworks []*hostNetwork + ) + + for _, intf := range inventory.Interfaces { + addresses := intf.IPV4Addresses + if !isIpv4 { + addresses = intf.IPV6Addresses + } + hasDefaultMetric, metric := getDefaultMetricForInterface(intf.Name, inventory.Routes, isIpv4) + for _, addrStr := range addresses { + addrIp, ipNet, err := net.ParseCIDR(addrStr) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse address %s for host %s", addrStr, lo.FromPtr(host.ID)) + } + hostNetworks = append(hostNetworks, &hostNetwork{ + hostId: *host.ID, + ipNet: ipNet, + }) + if hasDefaultMetric && lo.ContainsBy(connectedIps, func(ip net.IP) bool { return ip.Equal(addrIp) }) { + eligibleNodeIps = append(eligibleNodeIps, &eligibleNodeIp{ + nodeIp: addrIp, + ipNet: ipNet, + routeMetric: metric, + }) + } + } + } + if len(eligibleNodeIps) == 0 { + return nil, errors.Errorf("failed to create candidates for host %s", *host.ID) + } + return &hostCandidates{ + hostId: lo.FromPtr(host.ID), + hostNetworks: hostNetworks, + eligibleNodeIps: eligibleNodeIps, + }, nil +} + +func createHostsCandidates(hosts []*models.Host, connectivity *Connectivity, isIpv4 bool) (candidates []*hostCandidates, err error) { + for _, h := range hosts { + c, err := createHostCandidates(h, connectivity, isIpv4) + if err != nil { + return nil, err + } + candidates = append(candidates, c) + } + return +} + +type collisionDetector struct { + assignedNodeIps []*nodeIpCandidate + prohibitedNetworks []*hostNetwork +} + +func newCollisionDetector() *collisionDetector { + return &collisionDetector{} +} +func newCollisionDetectorForNodeIp(hostCandidates *hostCandidates, eligibleNodeIp *eligibleNodeIp) *collisionDetector { + // All host networks that the eligible Node IP does not belong to, are considered prohibited networks. It means + // that node-ip allocation on these networks should be avoided + prohibitedNetworks := lo.Filter(hostCandidates.hostNetworks, + func(hostNetwork *hostNetwork, _ int) bool { + return hostNetwork.ipNet.String() != eligibleNodeIp.ipNet.String() + }) + return &collisionDetector{ + assignedNodeIps: []*nodeIpCandidate{{ + hostId: hostCandidates.hostId, + ip: eligibleNodeIp.nodeIp, + }}, + prohibitedNetworks: prohibitedNetworks, + } +} + +func (c *collisionDetector) detectIpNetworkCollision(nodeIpCandidate *nodeIpCandidate, prohibitedNetworks []*hostNetwork) []error { + var errs []error + for _, prohibitedNetwork := range prohibitedNetworks { + if prohibitedNetwork.ipNet.Contains(nodeIpCandidate.ip) { + errs = append(errs, + errors.Errorf("IP assignment %s in host %s collides with prohibited network %s in host %s", + nodeIpCandidate.ip, nodeIpCandidate.hostId, prohibitedNetwork.ipNet.String(), prohibitedNetwork.hostId)) + } + } + return errs +} + +func (c *collisionDetector) detectIpsNetworksCollisions(assignedNodeIps []*nodeIpCandidate, prohibitedNetworks []*hostNetwork) []error { + var errs []error + for _, assignment := range assignedNodeIps { + errs = append(errs, c.detectIpNetworkCollision(assignment, prohibitedNetworks)...) + } + return errs +} + +func (c *collisionDetector) detectCollisions(other *collisionDetector) []error { + var errs []error + errs = append(append(errs, + c.detectIpsNetworksCollisions(c.assignedNodeIps, other.prohibitedNetworks)...), + c.detectIpsNetworksCollisions(other.assignedNodeIps, c.prohibitedNetworks)...) + return errs +} + +func (c *collisionDetector) merge(other *collisionDetector) *collisionDetector { + return &collisionDetector{ + assignedNodeIps: append(c.assignedNodeIps, other.assignedNodeIps...), + prohibitedNetworks: append(c.prohibitedNetworks, other.prohibitedNetworks...), + } +} + +func (c *collisionDetector) String() string { + return fmt.Sprintf("collision detector - assigned-node-ips [%s] prohibited-networks: [%s]", + strings.Join(lo.Map(c.assignedNodeIps, func(n *nodeIpCandidate, _ int) string { return n.ip.String() }), ","), + strings.Join(lo.Map(c.prohibitedNetworks, func(h *hostNetwork, _ int) string { return h.ipNet.String() }), ",")) +} + +type nodeIpAllocator struct { + candidates []*hostCandidates + clusterId strfmt.UUID + log logrus.FieldLogger +} + +func (n *nodeIpAllocator) sort(allocations []*allocationSolution) { + sort.SliceStable(allocations, func(i, j int) bool { + return allocations[i].routeMetric < allocations[j].routeMetric + }) +} + +// Allocate node IPs. Since the node IPs are dependent on each other, then all of them must be allocated +// through this allocation procedure. It tries to find an allocation when no conflict exists. +func (n *nodeIpAllocator) allocate(index int, parentContext *collisionDetector) (map[strfmt.UUID]*NodeIpAllocation, []error) { + if index == len(n.candidates) { + return make(map[strfmt.UUID]*NodeIpAllocation), nil + } + var ( + errs []error + allocationSolutions []*allocationSolution + h = n.candidates[index] + ) + for i := range h.eligibleNodeIps { + c := h.eligibleNodeIps[i] + localCollisionContext := newCollisionDetectorForNodeIp(h, c) + n.log.Debugf("local collision detector for host %s: %s", h.hostId, localCollisionContext) + if collisionErrs := localCollisionContext.detectCollisions(parentContext); len(collisionErrs) > 0 { + errs = append(errs, collisionErrs...) + continue + } + result, childErrs := n.allocate(index+1, parentContext.merge(localCollisionContext)) + if len(childErrs) > 0 { + errs = append(errs, childErrs...) + continue + } + result[h.hostId] = &NodeIpAllocation{ + NodeIp: c.nodeIp.String(), + HintIp: c.ipNet.IP.String(), + Cidr: c.ipNet.String(), + } + allocationSolutions = append(allocationSolutions, &allocationSolution{ + routeMetric: c.routeMetric, + nodeIpAllocations: result, + }) + } + if len(allocationSolutions) > 0 { + n.sort(allocationSolutions) + return allocationSolutions[0].nodeIpAllocations, nil + } + return nil, errs +} + +func (n *nodeIpAllocator) allocateNodeIps() (map[strfmt.UUID]*NodeIpAllocation, error) { + result, errs := n.allocate(0, newCollisionDetector()) + uniqErrs := lo.UniqBy(errs, func(e error) string { return e.Error() }) + err := stderrors.Join(uniqErrs...) + if err != nil { + n.log.WithError(err).Errorf("failed to allocate master node ips for cluster %s", n.clusterId) + return nil, err + } + return result, nil +} + +func GenerateNonePlatformAddressAllocation(cluster *common.Cluster, log logrus.FieldLogger) (map[strfmt.UUID]*NodeIpAllocation, error) { + addressFamilies, err := GetClusterAddressFamilies(cluster) + if err != nil { + return nil, errors.Wrapf(err, "failed tp get address families for cluster %s", lo.FromPtr(cluster.ID)) + } + isIpv4 := lo.Contains(addressFamilies, IPv4) + isIpv6 := lo.Contains(addressFamilies, IPv6) + if !(isIpv4 || isIpv6) { + return nil, errors.Errorf("No address family found for cluster %s", lo.FromPtr(cluster.ID)) + } + + if cluster.ConnectivityMajorityGroups == "" { + return nil, errors.Errorf("connectivity majority groups are missing for cluster %s", lo.FromPtr(cluster.ID)) + } + var connectivity Connectivity + err = json.Unmarshal([]byte(cluster.ConnectivityMajorityGroups), &connectivity) + if err != nil { + return nil, errors.Errorf("failed to unmarshal majority groups for cluster %s", lo.FromPtr(cluster.ID)) + } + if len(cluster.Hosts) != len(connectivity.L3ConnectedAddresses) { + return nil, errors.Errorf("not all the hosts are connected. Skipping address allocation for cluster %s", lo.FromPtr(cluster.ID)) + } + candidates, err := createHostsCandidates(cluster.Hosts, &connectivity, isIpv4) + if err != nil { + return nil, errors.Wrapf(err, "failed to create hosts candidates for cluster %s", lo.FromPtr(cluster.ID)) + } + allocator := &nodeIpAllocator{ + candidates: candidates, + log: log, + clusterId: lo.FromPtr(cluster.ID), + } + allocations, err := allocator.allocateNodeIps() + if err != nil { + return nil, errors.Wrapf(err, "failed to perform node ips allocation for cluster %s", lo.FromPtr(cluster.ID)) + } + return allocations, nil +} diff --git a/internal/network/none_platform_node_ips_allocation_test.go b/internal/network/none_platform_node_ips_allocation_test.go new file mode 100644 index 000000000000..822936122cb0 --- /dev/null +++ b/internal/network/none_platform_node_ips_allocation_test.go @@ -0,0 +1,486 @@ +package network + +import ( + "encoding/json" + + "github.com/go-openapi/strfmt" + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/openshift/assisted-service/internal/common" + "github.com/openshift/assisted-service/models" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +var _ = Describe("none platform node ips allocation", func() { + createInterface := func(name string, addresses ...string) *models.Interface { + var ipv4Addresses, ipv6Addresses []string + for _, a := range addresses { + if IsIPV4CIDR(a) { + ipv4Addresses = append(ipv4Addresses, a) + } else { + ipv6Addresses = append(ipv6Addresses, a) + } + } + return &models.Interface{ + IPV4Addresses: ipv4Addresses, + IPV6Addresses: ipv6Addresses, + Name: name, + } + } + + createInterfaces := func(interfaces ...*models.Interface) []*models.Interface { + return interfaces + } + createRoute := func(intf, dest, gw string, family, metric int32) *models.Route { + return &models.Route{ + Destination: dest, + Family: family, + Interface: intf, + Metric: metric, + Gateway: gw, + } + } + createDefaultRoute := func(intf string, family int32) *models.Route { + var dest, gw string + if family == unix.AF_INET { + dest = "0.0.0.0" + gw = "1.1.1.1" + } else { + dest = "::" + gw = "1:1:1::1" + } + return createRoute( + intf, + dest, + gw, + family, + 100) + } + createRoutes := func(routes ...*models.Route) []*models.Route { + return routes + } + createInventory := func(interfaces []*models.Interface, routes []*models.Route) string { + inventory := models.Inventory{ + Interfaces: interfaces, + Routes: routes, + } + b, err := json.Marshal(&inventory) + Expect(err).ToNot(HaveOccurred()) + return string(b) + } + createHost := func(inventory string, role models.HostRole) *models.Host { + id := strfmt.UUID(uuid.New().String()) + return &models.Host{ + ID: &id, + Inventory: inventory, + SuggestedRole: role, + Role: role, + } + } + generateL3ConnectedAddresses := func(connectedAddresses map[strfmt.UUID][]string) string { + majorityGroups := &Connectivity{ + L3ConnectedAddresses: connectedAddresses, + } + tmp, err := json.Marshal(majorityGroups) + Expect(err).ToNot(HaveOccurred()) + return string(tmp) + } + createCluster := func(hosts []*models.Host, connectivity string, networking common.TestNetworking) *common.Cluster { + id := strfmt.UUID(uuid.New().String()) + return &common.Cluster{ + Cluster: models.Cluster{ + ID: &id, + Hosts: hosts, + ConnectivityMajorityGroups: connectivity, + ClusterNetworks: networking.ClusterNetworks, + ServiceNetworks: networking.ServiceNetworks, + }, + } + } + It("no hosts, no connectivity", func() { + cluster := createCluster(nil, "", common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).To(HaveOccurred()) + Expect(allocation).To(BeNil()) + }) + It("only masters - one network", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.5/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.6/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"1.2.3.5"}, + *h3.ID: {"1.2.3.6"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1.2.3.4", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h2.ID: { + NodeIp: "1.2.3.5", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h3.ID: { + NodeIp: "1.2.3.6", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + })) + }) + + It("only masters - one network, IPv6", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1001:db8::1/120")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET6))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "1001:db8::2/120")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET6))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth0", "1001:db8::3/120")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET6))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1001:db8::1"}, + *h2.ID: {"1001:db8::2"}, + *h3.ID: {"1001:db8::3"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv6Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1001:db8::1", + HintIp: "1001:db8::", + Cidr: "1001:db8::/120", + }, + *h2.ID: { + NodeIp: "1001:db8::2", + HintIp: "1001:db8::", + Cidr: "1001:db8::/120", + }, + *h3.ID: { + NodeIp: "1001:db8::3", + HintIp: "1001:db8::", + Cidr: "1001:db8::/120", + }, + })) + }) + + It("only masters - two network", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.5/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"1.2.3.5"}, + *h3.ID: {"5.6.7.8"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1.2.3.4", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h2.ID: { + NodeIp: "1.2.3.5", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h3.ID: { + NodeIp: "5.6.7.8", + HintIp: "5.6.7.0", + Cidr: "5.6.7.0/24", + }, + })) + }) + It("only masters - two network, with overlapping networks", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.5/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth1", "1.2.3.6/24"), createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"1.2.3.5"}, + *h3.ID: {"5.6.7.8", "1.2.3.6"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1.2.3.4", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h2.ID: { + NodeIp: "1.2.3.5", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h3.ID: { + NodeIp: "1.2.3.6", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + })) + }) + It("only masters - two network, with overlapping networks. node-ip not connected", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.5/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth1", "1.2.3.6/24"), createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"1.2.3.5"}, + *h3.ID: {"5.6.7.8"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).To(HaveOccurred()) + Expect(allocation).To(BeNil()) + }) + It("only masters - two network, with overlapping networks, with colliding networks", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.7/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth1", "1.2.3.6/24"), createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"5.6.7.7"}, + *h3.ID: {"5.6.7.8", "1.2.3.6"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).To(HaveOccurred()) + Expect(allocation).To(BeNil()) + }) + It("only masters - three network, with overlapping networks", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.7/24"), createInterface("eth1", "6.7.8.9/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth1", "1.2.3.6/24"), createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + hosts := []*models.Host{ + h1, + h2, + h3, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"5.6.7.7", "6.7.8.9"}, + *h3.ID: {"5.6.7.8", "1.2.3.6"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1.2.3.4", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h2.ID: { + NodeIp: "6.7.8.9", + HintIp: "6.7.8.0", + Cidr: "6.7.8.0/24", + }, + *h3.ID: { + NodeIp: "1.2.3.6", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + })) + }) + It("only masters and workers - one network", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.5/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.6/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h4 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.7/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleWorker) + h5 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleWorker) + hosts := []*models.Host{ + h1, + h2, + h3, + h4, + h5, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"1.2.3.5"}, + *h3.ID: {"1.2.3.6"}, + *h4.ID: {"1.2.3.7"}, + *h5.ID: {"1.2.3.8"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1.2.3.4", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h2.ID: { + NodeIp: "1.2.3.5", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h3.ID: { + NodeIp: "1.2.3.6", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h4.ID: { + NodeIp: "1.2.3.7", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h5.ID: { + NodeIp: "1.2.3.8", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + })) + }) + It("masters and workers - three network, with overlapping networks - with collisions", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.7/24"), createInterface("eth1", "6.7.8.9/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth1", "1.2.3.6/24"), createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + h4 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.9/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleWorker) + h5 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.10/24"), createInterface("eth1", "1.2.3.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleWorker) + hosts := []*models.Host{ + h1, + h2, + h3, + h4, + h5, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"5.6.7.7", "6.7.8.9"}, + *h3.ID: {"5.6.7.8", "1.2.3.6"}, + *h4.ID: {"5.6.7.9"}, + *h5.ID: {"5.6.7.10", "1.2.3.8"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).To(HaveOccurred()) + Expect(allocation).To(BeNil()) + }) + It("masters and workers - three network, with overlapping networks - no collisions", func() { + h1 := createHost(createInventory(createInterfaces(createInterface("eth0", "1.2.3.4/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleMaster) + h2 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.7/24"), createInterface("eth1", "6.7.8.9/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + h3 := createHost(createInventory(createInterfaces(createInterface("eth1", "1.2.3.6/24"), createInterface("eth0", "5.6.7.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleMaster) + h4 := createHost(createInventory(createInterfaces(createInterface("eth0", "6.7.8.10/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET))), models.HostRoleWorker) + h5 := createHost(createInventory(createInterfaces(createInterface("eth0", "5.6.7.10/24"), createInterface("eth1", "1.2.3.8/24")), + createRoutes(createDefaultRoute("eth0", unix.AF_INET), createDefaultRoute("eth1", unix.AF_INET))), models.HostRoleWorker) + hosts := []*models.Host{ + h1, + h2, + h3, + h4, + h5, + } + connectivity := map[strfmt.UUID][]string{ + *h1.ID: {"1.2.3.4"}, + *h2.ID: {"5.6.7.7", "6.7.8.9"}, + *h3.ID: {"5.6.7.8", "1.2.3.6"}, + *h4.ID: {"6.7.8.10"}, + *h5.ID: {"5.6.7.10", "1.2.3.8"}, + } + cluster := createCluster(hosts, generateL3ConnectedAddresses(connectivity), common.TestIPv4Networking) + allocation, err := GenerateNonePlatformAddressAllocation(cluster, logrus.New()) + Expect(err).ToNot(HaveOccurred()) + Expect(allocation).To(Equal(map[strfmt.UUID]*NodeIpAllocation{ + *h1.ID: { + NodeIp: "1.2.3.4", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h2.ID: { + NodeIp: "6.7.8.9", + HintIp: "6.7.8.0", + Cidr: "6.7.8.0/24", + }, + *h3.ID: { + NodeIp: "1.2.3.6", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + *h4.ID: { + NodeIp: "6.7.8.10", + HintIp: "6.7.8.0", + Cidr: "6.7.8.0/24", + }, + *h5.ID: { + NodeIp: "1.2.3.8", + HintIp: "1.2.3.0", + Cidr: "1.2.3.0/24", + }, + })) + }) +}) diff --git a/internal/provider/none/installConfig.go b/internal/provider/none/installConfig.go index 4abe26e3d4b6..ddb27950c318 100644 --- a/internal/provider/none/installConfig.go +++ b/internal/provider/none/installConfig.go @@ -5,10 +5,40 @@ import ( "github.com/openshift/assisted-service/internal/common" "github.com/openshift/assisted-service/internal/host/hostutil" "github.com/openshift/assisted-service/internal/installcfg" + "github.com/openshift/assisted-service/internal/network" "github.com/openshift/assisted-service/internal/provider" "github.com/openshift/assisted-service/models" + "github.com/samber/lo" ) +func (p noneProvider) replaceMachineNetworkIfNeeded(cluster *common.Cluster, cfg *installcfg.InstallerConfigBaremetal) { + bootstrapHost := common.GetBootstrapHost(cluster) + if bootstrapHost == nil { + p.Log.Warnf("GetBootstrapHost: failed to get bootstrap host for cluster %s", lo.FromPtr(cluster.ID)) + return + } + nodeIpAllocations, err := network.GenerateNonePlatformAddressAllocation(cluster, p.Log) + if err != nil { + p.Log.WithError(err).Warnf("failed to get node ip allocations for cluster %s", lo.FromPtr(cluster.ID)) + return + } + allocation, ok := nodeIpAllocations[lo.FromPtr(bootstrapHost.ID)] + if !ok { + p.Log.Warnf("node ip allocation for bootstrap host %s in cluster %s is missing", lo.FromPtr(bootstrapHost.ID), lo.FromPtr(cluster.ID)) + return + } + isIpv4 := network.IsIPV4CIDR(allocation.Cidr) + for i := range cfg.Networking.MachineNetwork { + if network.IsIPV4CIDR(cfg.Networking.MachineNetwork[i].Cidr) == isIpv4 { + if cfg.Networking.MachineNetwork[i].Cidr != allocation.Cidr { + p.Log.Infof("Replacing machine network %s with %s", cfg.Networking.MachineNetwork[i].Cidr, allocation.Cidr) + cfg.Networking.MachineNetwork[i].Cidr = allocation.Cidr + } + break + } + } +} + func (p noneProvider) AddPlatformToInstallConfig(cfg *installcfg.InstallerConfigBaremetal, cluster *common.Cluster) error { cfg.Platform = installcfg.Platform{ None: &installcfg.PlatformNone{}, @@ -29,6 +59,8 @@ func (p noneProvider) AddPlatformToInstallConfig(cfg *installcfg.InstallerConfig if bootstrap != nil { cfg.BootstrapInPlace = &installcfg.BootstrapInPlace{InstallationDisk: hostutil.GetHostInstallationPath(bootstrap)} } + } else { + p.replaceMachineNetworkIfNeeded(cluster, cfg) } return nil