diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index dae1aaf71e..39a2a40f19 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -456,7 +456,7 @@ func (oc *DefaultNetworkController) run(_ context.Context) error { if config.OVNKubernetesFeature.EnableEgressIP { // This is probably the best starting order for all egress IP handlers. - // WatchEgressIPNamespaces and WatchEgressIPPods only use the informer + // WatchEgressIPPods and WatchEgressIPNamespaces only use the informer // cache to retrieve the egress IPs when determining if namespace/pods // match. It is thus better if we initialize them first and allow // WatchEgressNodes / WatchEgressIP to initialize after. Those handlers @@ -465,10 +465,14 @@ func (oc *DefaultNetworkController) run(_ context.Context) error { // risk performing a bunch of modifications on the EgressIP objects when // we restart and then have these handlers act on stale data when they // sync. - if err := WithSyncDurationMetric("egress ip namespace", oc.WatchEgressIPNamespaces); err != nil { + // Initialize WatchEgressIPPods before WatchEgressIPNamespaces to ensure + // that no pod events are missed by the EgressIPController. It's acceptable + // to miss a namespace event, as it will be handled indirectly through + // the pod delete event within that namespace. + if err := WithSyncDurationMetric("egress ip pod", oc.WatchEgressIPPods); err != nil { return err } - if err := WithSyncDurationMetric("egress ip pod", oc.WatchEgressIPPods); err != nil { + if err := WithSyncDurationMetric("egress ip namespace", oc.WatchEgressIPNamespaces); err != nil { return err } if err := WithSyncDurationMetric("egress node", oc.WatchEgressNodes); err != nil { @@ -1183,13 +1187,13 @@ func (h *defaultNetworkControllerEventHandler) SyncFunc(objs []interface{}) erro case factory.EgressFirewallType: syncFunc = h.oc.syncEgressFirewall - case factory.EgressIPNamespaceType: + case factory.EgressIPPodType: syncFunc = h.oc.eIPC.syncEgressIPs case factory.EgressNodeType: syncFunc = h.oc.eIPC.initClusterEgressPolicies - case factory.EgressIPPodType, + case factory.EgressIPNamespaceType, factory.EgressIPType: syncFunc = nil diff --git a/go-controller/pkg/ovn/egressip.go b/go-controller/pkg/ovn/egressip.go index 709e06b3f9..194bb9989a 100644 --- a/go-controller/pkg/ovn/egressip.go +++ b/go-controller/pkg/ovn/egressip.go @@ -788,25 +788,7 @@ func (e *EgressIPController) addPodEgressIPAssignments(ni util.NetInfo, name str if len(statusAssignments) == 0 { return nil } - // We need to proceed with add only under two conditions - // 1) egressNode present in at least one status is local to this zone - // (NOTE: The relation between egressIPName and nodeName is 1:1 i.e in the same object the given node will be present only in one status) - // 2) the pod being added is local to this zone - proceed := false - for _, status := range statusAssignments { - e.nodeZoneState.LockKey(status.Node) - isLocalZoneEgressNode, loadedEgressNode := e.nodeZoneState.Load(status.Node) - if loadedEgressNode && isLocalZoneEgressNode { - proceed = true - e.nodeZoneState.UnlockKey(status.Node) - break - } - e.nodeZoneState.UnlockKey(status.Node) - } - if !proceed && !e.isPodScheduledinLocalZone(pod) { - return nil // nothing to do if none of the status nodes are local to this master and pod is also remote - } - var remainingAssignments []egressipv1.EgressIPStatusItem + var remainingAssignments, staleAssignments []egressipv1.EgressIPStatusItem nadName := ni.GetNetworkName() if ni.IsUserDefinedNetwork() { nadNames := ni.GetNADs() @@ -836,15 +818,21 @@ func (e *EgressIPController) addPodEgressIPAssignments(ni util.NetInfo, name str podIPs: podIPs, network: ni, } - e.podAssignment.Store(podKey, podState) } else if podState.egressIPName == name || podState.egressIPName == "" { // We do the setup only if this egressIP object is the one serving this pod OR // podState.egressIPName can be empty if no re-routes were found in // syncPodAssignmentCache for the existing pod, we will treat this case as a new add for _, status := range statusAssignments { - if exists := podState.egressStatuses.contains(status); !exists { + // Add the status if it's not already in the cache, or if it exists but is in pending state + // (meaning it was populated during EIP sync and needs to be processed for the pod). + if value, exists := podState.egressStatuses.statusMap[status]; !exists || value == egressStatusStatePending { remainingAssignments = append(remainingAssignments, status) } + // Detect stale EIP status entries (same EgressIP reassigned to a different node) + // and queue the outdated entry for cleanup. + if staleStatus := podState.egressStatuses.hasStaleEIPStatus(status); staleStatus != nil { + staleAssignments = append(staleAssignments, *staleStatus) + } } podState.podIPs = podIPs podState.egressIPName = name @@ -866,6 +854,36 @@ func (e *EgressIPController) addPodEgressIPAssignments(ni util.NetInfo, name str podState.standbyEgressIPNames.Insert(name) return nil } + for _, staleStatus := range staleAssignments { + klog.V(2).Infof("Deleting stale pod egress IP status: %v for EgressIP: %s and pod: %s/%s/%v", staleStatus, name, pod.Namespace, pod.Name, podIPNets) + err = e.deletePodEgressIPAssignments(ni, name, []egressipv1.EgressIPStatusItem{staleStatus}, pod) + if err != nil { + klog.Warningf("Failed to delete stale EgressIP status %s/%v for pod %s: %v", name, staleStatus, podKey, err) + } + delete(podState.egressStatuses.statusMap, staleStatus) + } + // We store podState into podAssignment cache at this place for two reasons. + // 1. When podAssignmentState is newly created. + // 2. deletePodEgressIPAssignments might clean the podAssignment cache, make sure we add it back. + e.podAssignment.Store(podKey, podState) + // We need to proceed with add only under two conditions + // 1) egressNode present in at least one status is local to this zone + // (NOTE: The relation between egressIPName and nodeName is 1:1 i.e in the same object the given node will be present only in one status) + // 2) the pod being added is local to this zone + proceed := false + for _, status := range statusAssignments { + e.nodeZoneState.LockKey(status.Node) + isLocalZoneEgressNode, loadedEgressNode := e.nodeZoneState.Load(status.Node) + if loadedEgressNode && isLocalZoneEgressNode { + proceed = true + e.nodeZoneState.UnlockKey(status.Node) + break + } + e.nodeZoneState.UnlockKey(status.Node) + } + if !proceed && !e.isPodScheduledinLocalZone(pod) { + return nil // nothing to do if none of the status nodes are local to this master and pod is also remote + } for _, status := range remainingAssignments { klog.V(2).Infof("Adding pod egress IP status: %v for EgressIP: %s and pod: %s/%s/%v", status, name, pod.Namespace, pod.Name, podIPNets) nodesToLock := []string{status.Node, pod.Spec.NodeName} @@ -1155,6 +1173,8 @@ type egressIPCache struct { egressLocalNodesCache sets.Set[string] // egressIP IP -> assigned node name egressIPIPToNodeCache map[string]string + // egressIP name -> egress IP -> assigned node name + egressIPToAssignedNodes map[string]map[string]string // node name -> network name -> redirect IPs egressNodeRedirectsCache nodeNetworkRedirects // network name -> OVN cluster router name @@ -1594,6 +1614,14 @@ func (e *EgressIPController) syncPodAssignmentCache(egressIPCache egressIPCache) } } + // populate podState.egressStatuses with assigned node for active egressIP IPs. + if podState.egressIPName == egressIPName { + for egressIPIP, nodeName := range egressIPCache.egressIPToAssignedNodes[egressIPName] { + podState.egressStatuses.statusMap[egressipv1.EgressIPStatusItem{ + EgressIP: egressIPIP, Node: nodeName}] = egressStatusStatePending + } + } + e.podAssignment.Store(podKey, podState) return nil }); err != nil { @@ -1611,6 +1639,21 @@ func (e *EgressIPController) syncPodAssignmentCache(egressIPCache egressIPCache) // It also removes stale nexthops from router policies used by EgressIPs. // Upon failure, it may be invoked multiple times in order to avoid a pod restart. func (e *EgressIPController) syncStaleEgressReroutePolicy(cache egressIPCache) error { + // limit Nodes only to egress node(s) for the EgressIP name + limitToValidEgressNodes := func(eipName string, nodeRedirectCache map[string]redirectIPs) map[string]redirectIPs { + filteredEgressNodesRedirectsCache := make(map[string]redirectIPs, 0) + egressNodeNames, ok := cache.egressIPNameToAssignedNodes[eipName] + if !ok { + return filteredEgressNodesRedirectsCache + } + for _, egressNode := range egressNodeNames { + if nodeRedirect, ok := nodeRedirectCache[egressNode]; ok { + filteredEgressNodesRedirectsCache[egressNode] = nodeRedirect + } + } + return filteredEgressNodesRedirectsCache + } + for eipName, networkCache := range cache.egressIPNameToPods { for networkName, data := range networkCache { logicalRouterPolicyStaleNexthops := []*nbdb.LogicalRouterPolicy{} @@ -1619,11 +1662,6 @@ func (e *EgressIPController) syncStaleEgressReroutePolicy(cache egressIPCache) e if item.Priority != types.EgressIPReroutePriority || item.ExternalIDs[libovsdbops.NetworkKey.String()] != networkName { return false } - networkNodeRedirectCache, ok := cache.egressNodeRedirectsCache.cache[networkName] - if !ok || len(networkNodeRedirectCache) == 0 { - klog.Infof("syncStaleEgressReroutePolicy found invalid logical router policy (UUID: %s) because no assigned Nodes for EgressIP %s", item.UUID, eipName) - return true - } extractedEgressIPName, _ := getEIPLRPObjK8MetaData(item.ExternalIDs) if extractedEgressIPName == "" { klog.Errorf("syncStaleEgressReroutePolicy found logical router policy (UUID: %s) with invalid meta data associated with network %s", item.UUID, networkName) @@ -1634,6 +1672,11 @@ func (e *EgressIPController) syncStaleEgressReroutePolicy(cache egressIPCache) e _, ok := cache.egressIPNameToPods[extractedEgressIPName] return !ok } + networkNodeRedirectCache := limitToValidEgressNodes(eipName, cache.egressNodeRedirectsCache.cache[networkName]) + if len(networkNodeRedirectCache) == 0 { + klog.Infof("syncStaleEgressReroutePolicy deleting invalid logical router policy %q because there are no existing nodes assigned to its EgressIP %s", item.UUID, eipName) + return true + } splitMatch := strings.Split(item.Match, " ") podIPStr := splitMatch[len(splitMatch)-1] podIP := net.ParseIP(podIPStr) @@ -1689,13 +1732,13 @@ func (e *EgressIPController) syncStaleEgressReroutePolicy(cache egressIPCache) e // Update Logical Router Policies that have stale nexthops. Notice that we must do this separately // because logicalRouterPolicyStaleNexthops must be populated first for _, staleNextHopLogicalRouterPolicy := range logicalRouterPolicyStaleNexthops { - if staleNextHopLogicalRouterPolicy.Nexthop == nil { - continue - } - klog.Infof("syncStaleEgressReroutePolicy will remove stale nexthops for LRP %q for network %s: %s", - staleNextHopLogicalRouterPolicy.UUID, networkName, *staleNextHopLogicalRouterPolicy.Nexthop) + klog.Infof("syncStaleEgressReroutePolicy will remove stale nexthops for LRP %q for network %s: %v", + staleNextHopLogicalRouterPolicy.UUID, networkName, staleNextHopLogicalRouterPolicy.Nexthops) + } + // nothing to do if there's no stale next hops + if len(logicalRouterPolicyStaleNexthops) == 0 { + continue } - err = libovsdbops.DeleteNextHopsFromLogicalRouterPolicies(e.nbClient, cache.networkToRouter[networkName], logicalRouterPolicyStaleNexthops...) if err != nil { return fmt.Errorf("unable to remove stale next hops from logical router policies for network %s: %v", networkName, err) @@ -1874,28 +1917,37 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { r := redirectIPs{} mgmtPort := &nbdb.LogicalSwitchPort{Name: ni.GetNetworkScopedK8sMgmtIntfName(node.Name)} mgmtPort, err := libovsdbops.GetLogicalSwitchPort(e.nbClient, mgmtPort) - if err != nil { - // if switch port isnt created, we can assume theres nothing to sync - if errors.Is(err, libovsdbclient.ErrNotFound) { - continue - } + // return if error is anything other than not found to allow retry + if err != nil && !errors.Is(err, libovsdbclient.ErrNotFound) { return cache, fmt.Errorf("failed to find management port for node %s: %v", node.Name, err) } - mgmtPortAddresses := mgmtPort.GetAddresses() - if len(mgmtPortAddresses) == 0 { - return cache, fmt.Errorf("management switch port %s for node %s does not contain any addresses", ni.GetNetworkScopedK8sMgmtIntfName(node.Name), node.Name) - } - // assuming only one IP per IP family - for _, mgmtPortAddress := range mgmtPortAddresses { - mgmtPortAddressesStr := strings.Fields(mgmtPortAddress) - mgmtPortIP := net.ParseIP(mgmtPortAddressesStr[1]) - if utilnet.IsIPv6(mgmtPortIP) { - if ip := mgmtPortIP.To16(); ip != nil { - r.v6MgtPort = ip.String() + // if management port is available, gather the data. If it's not available, OVN constructs that depend on a deleted + // management port IP will fail and be cleaned up in sync LRPs func. + if mgmtPort != nil { + mgmtPortAddresses := mgmtPort.GetAddresses() + if len(mgmtPortAddresses) == 0 { + return cache, fmt.Errorf("management switch port %s for node %s does not contain any addresses", ni.GetNetworkScopedK8sMgmtIntfName(node.Name), node.Name) + } + // Extract at most one IP per family; entries are "MAC IP [IP ...]" + for _, macPlusIPs := range mgmtPortAddresses { + parts := strings.Fields(macPlusIPs) + if len(parts) < 2 { + continue // no IPs } - } else { - if ip := mgmtPortIP.To4(); ip != nil { - r.v4MgtPort = ip.String() + for _, ipStr := range parts[1:] { + ip := net.ParseIP(ipStr) + if ip == nil { + continue + } + if utilnet.IsIPv6(ip) { + if r.v6MgtPort == "" && ip.To16() != nil { + r.v6MgtPort = ip.String() + } + } else { + if r.v4MgtPort == "" && ip.To4() != nil { + r.v4MgtPort = ip.String() + } + } } } } @@ -1951,6 +2003,9 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { // egressIP IP -> node name. Assigned node for EIP. egressIPIPNodeCache := make(map[string]string, 0) cache.egressIPIPToNodeCache = egressIPIPNodeCache + // egressIP name -> egressIP IP -> node name. + egressIPToAssignedNodes := make(map[string]map[string]string, 0) + cache.egressIPToAssignedNodes = egressIPToAssignedNodes cache.markCache = make(map[string]string) egressIPs, err := e.watchFactory.GetEgressIPs() if err != nil { @@ -1964,6 +2019,7 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { cache.markCache[egressIP.Name] = mark.String() egressIPsCache[egressIP.Name] = make(map[string]selectedPods, 0) egressIPNameNodesCache[egressIP.Name] = make([]string, 0, len(egressIP.Status.Items)) + egressIPToAssignedNodes[egressIP.Name] = make(map[string]string, 0) for _, status := range egressIP.Status.Items { eipIP := net.ParseIP(status.EgressIP) if eipIP == nil { @@ -1971,6 +2027,7 @@ func (e *EgressIPController) generateCacheForEgressIP() (egressIPCache, error) { continue } egressIPIPNodeCache[eipIP.String()] = status.Node + egressIPToAssignedNodes[egressIP.Name][eipIP.String()] = status.Node if localZoneNodes.Has(status.Node) { egressLocalNodesCache.Insert(status.Node) } @@ -2227,9 +2284,18 @@ func InitClusterEgressPolicies(nbClient libovsdbclient.Client, addressSetFactory return nil } +// egressStatusStatePending marks entries populated during EIP sync and +// indicates they must be reconciled again for the pod. +const egressStatusStatePending = "pending" + type statusMap map[egressipv1.EgressIPStatusItem]string type egressStatuses struct { + // statusMap tracks per EIP status assignment for a pod. + // Key: egressipv1.EgressIPStatusItem {EgressIP, Node} + // Values: + // "" -> applied/reconciled + // egressStatusStatePending -> populated during EIP sync, pending reconcile. statusMap } @@ -2241,6 +2307,21 @@ func (e egressStatuses) contains(potentialStatus egressipv1.EgressIPStatusItem) return false } +// hasStaleEIPStatus checks for stale EIP status entries already in cache. +// This addresses the race condition where an EIP is reassigned to a different node +// but the cache still contains the old assignment, leading to stale SNAT/LRP entries. +func (e egressStatuses) hasStaleEIPStatus(potentialStatus egressipv1.EgressIPStatusItem) *egressipv1.EgressIPStatusItem { + var staleStatus *egressipv1.EgressIPStatusItem + for status := range e.statusMap { + if status.EgressIP == potentialStatus.EgressIP && + status.Node != potentialStatus.Node { + staleStatus = &egressipv1.EgressIPStatusItem{EgressIP: status.EgressIP, Node: status.Node} + break + } + } + return staleStatus +} + func (e egressStatuses) delete(deleteStatus egressipv1.EgressIPStatusItem) { delete(e.statusMap, deleteStatus) } diff --git a/go-controller/pkg/ovn/egressip_test.go b/go-controller/pkg/ovn/egressip_test.go index 131c63d354..cb15a3de1d 100644 --- a/go-controller/pkg/ovn/egressip_test.go +++ b/go-controller/pkg/ovn/egressip_test.go @@ -58,6 +58,7 @@ const ( v6GatewayIP = "ae70::1" v6Node1Subnet = "ae70::66/64" v6Node2Subnet = "be70::66/64" + v6Node3Subnet = "ce70::66/64" v4ClusterSubnet = "10.128.0.0/14" v4Node1Subnet = "10.128.0.0/16" v4Node2Subnet = "10.90.0.0/16" @@ -338,9 +339,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -553,9 +554,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -796,9 +797,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1134,9 +1135,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1555,9 +1556,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1945,9 +1946,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2348,9 +2349,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&p.Pod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2765,9 +2766,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3218,9 +3219,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3493,9 +3494,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" i, n, _ := net.ParseCIDR(podV4IP + "/23") n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3731,9 +3732,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4014,9 +4015,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, false) } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4232,9 +4233,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4599,9 +4600,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }, } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4733,9 +4734,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }, } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4870,9 +4871,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, false) } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -5107,9 +5108,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, false) } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -5367,9 +5368,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }, } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -5546,9 +5547,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }, } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -5747,9 +5748,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.zone = "local" fakeOvn.controller.eIPC.zone = "local" } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -6226,9 +6227,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }, } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -6951,10 +6952,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -7306,8 +7307,8 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := fakeOvn.controller.lsManager.AddOrUpdateSwitch(node1.Name, []*net.IPNet{ovntest.MustParseIPNet(v4Node1Subnet)}, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(fakeOvn.controller.WatchPods()).To(gomega.Succeed()) - gomega.Expect(fakeOvn.controller.WatchEgressIPNamespaces()).To(gomega.Succeed()) gomega.Expect(fakeOvn.controller.WatchEgressIPPods()).To(gomega.Succeed()) + gomega.Expect(fakeOvn.controller.WatchEgressIPNamespaces()).To(gomega.Succeed()) gomega.Expect(fakeOvn.controller.WatchEgressNodes()).To(gomega.Succeed()) gomega.Expect(fakeOvn.controller.WatchEgressIP()).To(gomega.Succeed()) @@ -7690,10 +7691,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -7973,18 +7974,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" pas = getPodAssignmentState(&egressPod1) gomega.Expect(pas).NotTo(gomega.BeNil()) gomega.Expect(pas.egressIPName).To(gomega.Equal(egressIPName)) - gomega.Expect(pas.egressStatuses.statusMap).To(gomega.Equal(statusMap{})) + gomega.Expect(pas.egressStatuses.statusMap).To(gomega.HaveLen(2)) gomega.Expect(pas.standbyEgressIPNames.Has(egressIP2Name)).To(gomega.BeTrue()) - // reset egressStatuses for rest of the test to progress correctly - fakeOvn.controller.eIPC.podAssignment.LockKey(egressPod1Key) - podStatus, exists := fakeOvn.controller.eIPC.podAssignment.Load(egressPod1Key) - gomega.Expect(exists).To(gomega.BeTrue()) - podStatus.egressStatuses.statusMap[eip1Obj.Status.Items[0]] = "" - podStatus.egressStatuses.statusMap[eip1Obj.Status.Items[1]] = "" - fakeOvn.controller.eIPC.podAssignment.Store(egressPod1Key, podStatus) - fakeOvn.controller.eIPC.podAssignment.UnlockKey(egressPod1Key) - // delete the standby egressIP object to make sure the cache is updated err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Delete(context.TODO(), egressIP2Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -7995,8 +7987,8 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" eip1Obj, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Get(context.TODO(), eIP1.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(pas.egressStatuses.statusMap).To(gomega.HaveLen(2)) - g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[0]]).To(gomega.Equal("")) - g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[1]]).To(gomega.Equal("")) + g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[0]]).To(gomega.Equal(egressStatusStatePending)) + g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[1]]).To(gomega.Equal(egressStatusStatePending)) g.Expect(pas.standbyEgressIPNames.Has(egressIP2Name)).To(gomega.BeFalse()) }).Should(gomega.Succeed()) // add back the standby egressIP object @@ -8022,8 +8014,8 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" eip1Obj, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Get(context.TODO(), eIP1.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(pas.egressStatuses.statusMap).To(gomega.HaveLen(2)) - g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[0]]).To(gomega.Equal("")) - g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[1]]).To(gomega.Equal("")) + g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[0]]).To(gomega.Equal(egressStatusStatePending)) + g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[1]]).To(gomega.Equal(egressStatusStatePending)) g.Expect(pas.standbyEgressIPNames.Has(egressIP2Name)).To(gomega.BeTrue()) }).Should(gomega.Succeed()) gomega.Eventually(fakeOvn.fakeRecorder.Events).Should(gomega.Receive( @@ -8088,7 +8080,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" eip1Obj, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Get(context.TODO(), eIP1.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(pas.egressStatuses.statusMap).To(gomega.HaveLen(1)) - g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[0]]).To(gomega.Equal("")) + g.Expect(pas.egressStatuses.statusMap[eip1Obj.Status.Items[0]]).To(gomega.Equal(egressStatusStatePending)) g.Expect(pas.standbyEgressIPNames.Has(egressIP2Name)).To(gomega.BeTrue()) }).Should(gomega.Succeed()) // delete the first egressIP object and make sure the cache is updated @@ -8254,9 +8246,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Node{node}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -8526,9 +8518,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Node{node1, node2}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -8725,9 +8717,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Node{node1, node2}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -9046,9 +9038,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -9301,9 +9293,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }, ) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -9498,10 +9490,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := fakeOvn.controller.WatchPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -9686,9 +9678,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Node{node1, node2}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -10060,9 +10052,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Node{node1, node2}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -10468,10 +10460,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := fakeOvn.controller.WatchPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -10665,10 +10657,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err := fakeOvn.controller.WatchPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -10988,10 +10980,10 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" err = fakeOvn.controller.WatchPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -11154,124 +11146,619 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" ), ) - ginkgo.DescribeTable( - "should ensure EgressIP skips host-network and pending pods from namespace", - func(includeUnscheduled, includeScheduledButNoIP, includeHostNetwork, interconnect, isPodRemote bool) { - app.Action = func(*cli.Context) error { - config.Gateway.DisableSNATMultipleGWs = true - config.OVNKubernetesFeature.EnableInterconnect = interconnect + ginkgo.It("should update SNAT and LRP nexthops during simultaneous EIP failover and ovnkube-controller restart", func() { + app.Action = func(*cli.Context) error { + config.Gateway.DisableSNATMultipleGWs = true + config.OVNKubernetesFeature.EnableInterconnect = true - egressIP1 := "192.168.126.101" - node1IPv4 := "192.168.126.12" - node1IPv4CIDR := node1IPv4 + "/24" - node2IPv4 := "192.168.126.51" - node2IPv4CIDR := node2IPv4 + "/24" + egressIP := "192.168.126.101" + node1IPv4 := "192.168.126.12" + node1IPv4CIDR := node1IPv4 + "/24" + node1IPv4TranSwitchIP := "100.88.0.2/16" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + node2IPv4TranSwitchIP := "100.88.0.3" + node2IPv4TranSwitchIPCIDR := node2IPv4TranSwitchIP + "/16" + node3IPv4 := "192.168.126.61" + node3IPv4CIDR := node3IPv4 + "/24" + node3IPv4TranSwitchIP := "100.88.0.4/16" - egressPod3 := *newPodWithLabels(eipNamespace, "egress-pod3", node1Name, podV4IP, egressPodLabel) - egressPod4 := *newPodWithLabels(eipNamespace, "egress-pod4", node1Name, podV4IP2, egressPodLabel) - egressNamespace := newNamespace(eipNamespace) - annotations := map[string]string{ - "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\"}", node1IPv4CIDR), - "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v4Node1Subnet), - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop":"192.168.126.1"}}`, - "k8s.ovn.org/node-chassis-id": "79fdcfc4-6fe6-4cd3-8242-c0f85a4668ec", - util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), - "k8s.ovn.org/zone-name": "global", - } - node1 := getNodeObj(node1Name, annotations, map[string]string{}) - annotations = map[string]string{ - "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\"}", node2IPv4CIDR), - "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v4Node2Subnet), - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.51/24", "next-hop":"192.168.126.1"}}`, - "k8s.ovn.org/node-chassis-id": "89fdcfc4-6fe6-4cd3-8242-c0f85a4668ec", - util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), - "k8s.ovn.org/zone-name": "global", - } - if isPodRemote { - annotations["k8s.ovn.org/zone-name"] = "remote" - annotations["k8s.ovn.org/remote-zone-migrated"] = "remote" - } - node2 := getNodeObj(node2Name, annotations, map[string]string{}) + egressPod := *newPodWithLabels(eipNamespace, podName, node1Name, podV4IP, egressPodLabel) + egressPod2 := *newPodWithLabels(eipNamespace, podName2, node3Name, podV4IP2, egressPodLabel) + egressNamespace := newNamespace(eipNamespace) - eIP := egressipv1.EgressIP{ - ObjectMeta: newEgressIPMeta(egressIPName), - Spec: egressipv1.EgressIPSpec{ - EgressIPs: []string{egressIP1}, - NamespaceSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": egressNamespace.Name, - }, + nodes := getIPv4Nodes([]nodeInfo{{[]string{node1IPv4CIDR}, "global", node1IPv4TranSwitchIP}, + {[]string{node2IPv4CIDR}, "remote", node2IPv4TranSwitchIPCIDR}, {[]string{node3IPv4CIDR}, "remote", node3IPv4TranSwitchIP}}) + node1 := nodes[0] + node1.Labels = map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node1.Annotations["k8s.ovn.org/l3-gateway-config"] = `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop":"192.168.126.1"}}` + node1.Annotations["k8s.ovn.org/node-chassis-id"] = "79fdcfc4-6fe6-4cd3-8242-c0f85a4668ec" + node2 := nodes[1] + node2.Labels = map[string]string{ + "k8s.ovn.org/egress-assignable": "", + } + node2.Annotations["k8s.ovn.org/l3-gateway-config"] = `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.51/24", "next-hop":"192.168.126.1"}}` + node2.Annotations["k8s.ovn.org/node-chassis-id"] = "89fdcfc4-6fe6-4cd3-8242-c0f85a4668ec" + node3 := nodes[2] + node3.Annotations["k8s.ovn.org/l3-gateway-config"] = `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.75/24", "next-hop":"192.168.126.1"}}` + node3.Annotations["k8s.ovn.org/node-chassis-id"] = "99fdcfc4-6fe6-4cd3-8242-c0f85a4668ec" + + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMeta(egressIPName), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP}, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": egressNamespace.Name, }, }, - Status: egressipv1.EgressIPStatus{ - Items: []egressipv1.EgressIPStatusItem{}, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{ + { + Node: node1.Name, + EgressIP: egressIP, + }, }, - } - node1Switch := &nbdb.LogicalSwitch{ - UUID: node1.Name + "-UUID", - Name: node1.Name, - } - node2Switch := &nbdb.LogicalSwitch{ - UUID: node2.Name + "-UUID", - Name: node2.Name, - } - - pods := []corev1.Pod{egressPod3, egressPod4} - if includeUnscheduled { - unScheduledPod := corev1.Pod{ - ObjectMeta: newPodMeta(eipNamespace, "egress-pod", egressPodLabel), - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "containerName", - Image: "containerImage", - }, - }, + }, + } + node1Switch := &nbdb.LogicalSwitch{ + UUID: node1.Name + "-UUID", + Name: node1.Name, + } + node2Switch := &nbdb.LogicalSwitch{ + UUID: node2.Name + "-UUID", + Name: node2.Name, + } + node3Switch := &nbdb.LogicalSwitch{ + UUID: node3.Name + "-UUID", + Name: node3.Name, + } + logicalRouterOptions := map[string]string{ + "dynamic_neigh_routers": "false", + } + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + Name: types.OVNClusterRouter, + UUID: types.OVNClusterRouter + "-UUID", }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + UUID: types.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, + Options: logicalRouterOptions, }, - } - pods = append(pods, unScheduledPod) - } - var pendingScheduledPodWithNoIP *corev1.Pod - if includeScheduledButNoIP { - pendingScheduledPodWithNoIP = newPodWithLabels(eipNamespace, "egress-pod1", node2Name, "", egressPodLabel) - pendingScheduledPodWithNoIP.Status.Phase = corev1.PodPending - pods = append(pods, *pendingScheduledPodWithNoIP) - } - if includeHostNetwork { - hnPod := *newPodWithLabels(eipNamespace, "egress-pod2", node2Name, node2IPv4, egressPodLabel) - hnPod.Spec.HostNetwork = true - pods = append(pods, hnPod) - } - - dynamicNeighRouters := "true" - if config.OVNKubernetesFeature.EnableInterconnect { - dynamicNeighRouters = "false" - } - logicalRouterOptions := map[string]string{ - "dynamic_neigh_routers": dynamicNeighRouters, - } - expectedNatLogicalPort1 := "k8s-node1" - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalRouter{ - Name: types.OVNClusterRouter, - UUID: types.OVNClusterRouter + "-UUID", - // stale LRP reference to be removed by EIP controller sync function. - Policies: []string{"hostnet-pod-reroute-UUID1"}, - }, - &nbdb.LogicalRouter{ - Name: types.GWRouterPrefix + node1.Name, - UUID: types.GWRouterPrefix + node1.Name + "-UUID", - Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, - // stale NAT reference to be removed by EIP controller sync function. - Nat: []string{"hostnet-pod-egressip-nat-UUID1"}, - Options: logicalRouterOptions, - }, - &nbdb.LogicalRouter{ + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node2.Name, + UUID: types.GWRouterPrefix + node2.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node3.Name, + UUID: types.GWRouterPrefix + node3.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name + "-UUID"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name, + Networks: []string{nodeLogicalRouterIfAddrV4}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name, + Networks: []string{node2LogicalRouterIfAddrV4}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name, + Networks: []string{node3LogicalRouterIfAddrV4}, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node1Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node2Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node3Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node1Name + "-UUID", + Name: types.ExternalSwitchPrefix + node1Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name + "-UUID"}, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node2Name + "-UUID", + Name: types.ExternalSwitchPrefix + node2Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name + "-UUID"}, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node3Name + "-UUID", + Name: types.ExternalSwitchPrefix + node3Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name + "-UUID"}, + }, + node1Switch, + node2Switch, + node3Switch, + }, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIP}, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2, node3}, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{egressPod, egressPod2}, + }, + ) + + i, n, _ := net.ParseCIDR(podV4IP + "/23") + n.IP = i + fakeOvn.controller.logicalPortCache.add(&egressPod, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) + i, n, _ = net.ParseCIDR(podV4IP2 + "/23") + n.IP = i + fakeOvn.controller.logicalPortCache.add(&egressPod2, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) + + fakeOvn.controller.eIPC.nodeZoneState.Store(node1Name, true) + fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) + fakeOvn.controller.eIPC.nodeZoneState.Store(node3Name, false) + + err := fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressNodes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // To simulate an ovnkube-controller restart, update the EIP object with the newly assigned node. + // Then invoke reconcileEgressIP using only the updated EIP object to trigger the EgressIP add event. + eIP.Status = egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{ + { + Node: node2.Name, + EgressIP: egressIP, + }, + }, + } + err = fakeOvn.controller.eIPC.reconcileEgressIP(nil, &eIP) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(getPodAssignmentState(&egressPod)).NotTo(gomega.BeNil()) + + egressSVCServedPodsASv4, _ := buildEgressServiceAddressSets(nil) + egressIPServedPodsASv4, _ := buildEgressIPServedPodsAddressSets([]string{podV4IP}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + egressNodeIPsASv4, _ := buildEgressIPNodeAddressSets([]string{node1IPv4, node2IPv4, node3IPv4}) + + node1Switch.QOSRules = []string{"default-QoS-UUID"} + expectedDatabaseState := []libovsdbtest.TestData{ + &nbdb.LogicalRouterPolicy{ + Priority: types.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": types.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + }, + getNoReRouteReplyTrafficPolicy(types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName), + &nbdb.LogicalRouterPolicy{ + Priority: types.DefaultNoRereoutePriority, + Match: "ip4.src == 10.128.0.0/14 && ip4.dst == 10.128.0.0/14", + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: types.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == 10.128.0.0/14 && ip4.dst == %s", config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: types.EgressIPReroutePriority, + Match: fmt.Sprintf("ip4.src == %s", egressPod.Status.PodIP), + Action: nbdb.LogicalRouterPolicyActionReroute, + Nexthops: []string{node2IPv4TranSwitchIP}, + ExternalIDs: getEgressIPLRPReRouteDbIDs(eIP.Name, egressPod.Namespace, egressPod.Name, IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + UUID: "reroute-UUID1", + }, + &nbdb.LogicalRouter{ + Name: types.OVNClusterRouter, + UUID: types.OVNClusterRouter + "-UUID", + Policies: []string{"no-reroute-UUID", "no-reroute-service-UUID", "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic", "reroute-UUID1"}, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + UUID: types.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"pod-node-nat-UUID1"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node2.Name, + UUID: types.GWRouterPrefix + node2.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node3.Name, + UUID: types.GWRouterPrefix + node3.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name + "-UUID"}, + Options: logicalRouterOptions, + }, + &nbdb.NAT{ + UUID: "pod-node-nat-UUID1", + LogicalIP: podV4IP, + ExternalIP: node1IPv4, + ExternalIDs: nil, + Type: nbdb.NATTypeSNAT, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name, + Networks: []string{node2LogicalRouterIfAddrV4}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name, + Networks: []string{nodeLogicalRouterIfAddrV4}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name, + Networks: []string{node3LogicalRouterIfAddrV4}, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node1Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node2Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node3Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node1Name + "-UUID", + Name: types.ExternalSwitchPrefix + node1Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name + "-UUID"}, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node2Name + "-UUID", + Name: types.ExternalSwitchPrefix + node2Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name + "-UUID"}, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node3Name + "-UUID", + Name: types.ExternalSwitchPrefix + node3Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name + "-UUID"}, + }, + node1Switch, + node2Switch, + node3Switch, + getDefaultQoSRule(false, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName), + egressSVCServedPodsASv4, egressIPServedPodsASv4, egressNodeIPsASv4, + } + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + // Since we called reconcileEgressIP manually with new eIP status in the above step, so update EIP object with + // same status as well. + _, err = fakeOvn.fakeClient.EgressIPClient.K8sV1().EgressIPs().Update(context.TODO(), &eIP, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + // Now delete local zone egressPod and check LRP is removed. + err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(egressPod.Namespace).Delete(context.TODO(), egressPod.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + egressIPServedPodsASv4, _ = buildEgressIPServedPodsAddressSets([]string{}, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + node1Switch.QOSRules = []string{"default-QoS-UUID"} + expectedDatabaseState = []libovsdbtest.TestData{ + &nbdb.LogicalRouterPolicy{ + Priority: types.DefaultNoRereoutePriority, + Match: fmt.Sprintf("(ip4.src == $%s || ip4.src == $%s) && ip4.dst == $%s", + egressIPServedPodsASv4.Name, egressSVCServedPodsASv4.Name, egressNodeIPsASv4.Name), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "default-no-reroute-node-UUID", + Options: map[string]string{"pkt_mark": types.EgressIPNodeConnectionMark}, + ExternalIDs: getEgressIPLRPNoReRoutePodToNodeDbIDs(IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + }, + getNoReRouteReplyTrafficPolicy(types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName), + &nbdb.LogicalRouterPolicy{ + Priority: types.DefaultNoRereoutePriority, + Match: "ip4.src == 10.128.0.0/14 && ip4.dst == 10.128.0.0/14", + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToPodDbIDs(IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouterPolicy{ + Priority: types.DefaultNoRereoutePriority, + Match: fmt.Sprintf("ip4.src == 10.128.0.0/14 && ip4.dst == %s", config.Gateway.V4JoinSubnet), + Action: nbdb.LogicalRouterPolicyActionAllow, + UUID: "no-reroute-service-UUID", + ExternalIDs: getEgressIPLRPNoReRoutePodToJoinDbIDs(IPFamilyValueV4, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs(), + }, + &nbdb.LogicalRouter{ + Name: types.OVNClusterRouter, + UUID: types.OVNClusterRouter + "-UUID", + Policies: []string{"no-reroute-UUID", "no-reroute-service-UUID", "default-no-reroute-node-UUID", "default-no-reroute-reply-traffic"}, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + UUID: types.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"pod-node-nat-UUID1"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node2.Name, + UUID: types.GWRouterPrefix + node2.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node3.Name, + UUID: types.GWRouterPrefix + node3.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name + "-UUID"}, + Options: logicalRouterOptions, + }, + // we still expect the SNAT at the end because we are not watching pod events that would remove the SNAT on pod deletion + &nbdb.NAT{ + UUID: "pod-node-nat-UUID1", + LogicalIP: podV4IP, + ExternalIP: node1IPv4, + ExternalIDs: nil, + Type: nbdb.NATTypeSNAT, + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name, + Networks: []string{node2LogicalRouterIfAddrV4}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name, + Networks: []string{nodeLogicalRouterIfAddrV4}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node3.Name, + Networks: []string{node3LogicalRouterIfAddrV4}, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node1Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node2Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitchPort{ + UUID: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name + "-UUID", + Name: types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name, + Type: "router", + Options: map[string]string{ + libovsdbops.RouterPort: types.GWRouterToExtSwitchPrefix + "GR_" + node3Name, + "nat-addresses": "router", + "exclude-lb-vips-from-garp": "true", + }, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node1Name + "-UUID", + Name: types.ExternalSwitchPrefix + node1Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node1Name + "-UUID"}, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node2Name + "-UUID", + Name: types.ExternalSwitchPrefix + node2Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node2Name + "-UUID"}, + }, + &nbdb.LogicalSwitch{ + UUID: types.ExternalSwitchPrefix + node3Name + "-UUID", + Name: types.ExternalSwitchPrefix + node3Name, + Ports: []string{types.EXTSwitchToGWRouterPrefix + types.GWRouterPrefix + node3Name + "-UUID"}, + }, + node1Switch, + node2Switch, + node3Switch, + getDefaultQoSRule(false, types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName), + egressSVCServedPodsASv4, egressIPServedPodsASv4, egressNodeIPsASv4, + } + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.DescribeTable( + "should ensure EgressIP skips host-network and pending pods from namespace", + func(includeUnscheduled, includeScheduledButNoIP, includeHostNetwork, interconnect, isPodRemote bool) { + app.Action = func(*cli.Context) error { + config.Gateway.DisableSNATMultipleGWs = true + config.OVNKubernetesFeature.EnableInterconnect = interconnect + + egressIP1 := "192.168.126.101" + node1IPv4 := "192.168.126.12" + node1IPv4CIDR := node1IPv4 + "/24" + node2IPv4 := "192.168.126.51" + node2IPv4CIDR := node2IPv4 + "/24" + + egressPod3 := *newPodWithLabels(eipNamespace, "egress-pod3", node1Name, podV4IP, egressPodLabel) + egressPod4 := *newPodWithLabels(eipNamespace, "egress-pod4", node1Name, podV4IP2, egressPodLabel) + egressNamespace := newNamespace(eipNamespace) + annotations := map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\"}", node1IPv4CIDR), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v4Node1Subnet), + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.12/24", "next-hop":"192.168.126.1"}}`, + "k8s.ovn.org/node-chassis-id": "79fdcfc4-6fe6-4cd3-8242-c0f85a4668ec", + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node1IPv4CIDR), + "k8s.ovn.org/zone-name": "global", + } + node1 := getNodeObj(node1Name, annotations, map[string]string{}) + annotations = map[string]string{ + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{\"ipv4\": \"%s\"}", node2IPv4CIDR), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":\"%s\"}", v4Node2Subnet), + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"192.168.126.51/24", "next-hop":"192.168.126.1"}}`, + "k8s.ovn.org/node-chassis-id": "89fdcfc4-6fe6-4cd3-8242-c0f85a4668ec", + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", node2IPv4CIDR), + "k8s.ovn.org/zone-name": "global", + } + if isPodRemote { + annotations["k8s.ovn.org/zone-name"] = "remote" + annotations["k8s.ovn.org/remote-zone-migrated"] = "remote" + } + node2 := getNodeObj(node2Name, annotations, map[string]string{}) + + eIP := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMeta(egressIPName), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{egressIP1}, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": egressNamespace.Name, + }, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{}, + }, + } + node1Switch := &nbdb.LogicalSwitch{ + UUID: node1.Name + "-UUID", + Name: node1.Name, + } + node2Switch := &nbdb.LogicalSwitch{ + UUID: node2.Name + "-UUID", + Name: node2.Name, + } + + pods := []corev1.Pod{egressPod3, egressPod4} + if includeUnscheduled { + unScheduledPod := corev1.Pod{ + ObjectMeta: newPodMeta(eipNamespace, "egress-pod", egressPodLabel), + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "containerName", + Image: "containerImage", + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + pods = append(pods, unScheduledPod) + } + var pendingScheduledPodWithNoIP *corev1.Pod + if includeScheduledButNoIP { + pendingScheduledPodWithNoIP = newPodWithLabels(eipNamespace, "egress-pod1", node2Name, "", egressPodLabel) + pendingScheduledPodWithNoIP.Status.Phase = corev1.PodPending + pods = append(pods, *pendingScheduledPodWithNoIP) + } + if includeHostNetwork { + hnPod := *newPodWithLabels(eipNamespace, "egress-pod2", node2Name, node2IPv4, egressPodLabel) + hnPod.Spec.HostNetwork = true + pods = append(pods, hnPod) + } + + dynamicNeighRouters := "true" + if config.OVNKubernetesFeature.EnableInterconnect { + dynamicNeighRouters = "false" + } + logicalRouterOptions := map[string]string{ + "dynamic_neigh_routers": dynamicNeighRouters, + } + expectedNatLogicalPort1 := "k8s-node1" + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + Name: types.OVNClusterRouter, + UUID: types.OVNClusterRouter + "-UUID", + // stale LRP reference to be removed by EIP controller sync function. + Policies: []string{"hostnet-pod-reroute-UUID1"}, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + UUID: types.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, + // stale NAT reference to be removed by EIP controller sync function. + Nat: []string{"hostnet-pod-egressip-nat-UUID1"}, + Options: logicalRouterOptions, + }, + &nbdb.LogicalRouter{ Name: types.GWRouterPrefix + node2.Name, UUID: types.GWRouterPrefix + node2.Name + "-UUID", Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node2.Name + "-UUID"}, @@ -11370,9 +11857,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.eIPC.nodeZoneState.Store(node2Name, false) } - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -11827,9 +12314,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace, *egressNamespace2}, }) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -12266,9 +12753,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod2, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -13057,9 +13544,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" Items: []corev1.Namespace{*egressNamespace}, }, ) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -13364,9 +13851,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" n.IP = i fakeOvn.controller.logicalPortCache.add(&egressPod1, "", types.DefaultNetworkName, "", nil, []*net.IPNet{n}) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -13880,6 +14367,247 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" }) ginkgo.Context("Sync", func() { + ginkgo.DescribeTable("remove invalid next hop from LRP", func(isV6 bool) { + // scenario is OVNDB is config'd with pod is local and EIP egress is local and remote. + // The EgressIPs assigned Nodes is however different from the OVNDB config therefore we expect Sync to + // reconcile and update the local pod LRP (attached to cluster router) next hops to reflect the current state. + // removes invalid next hop from LRP. Controller may have missed the signal that the EIP moved to another node. + app.Action = func(*cli.Context) error { + config.OVNKubernetesFeature.EnableInterconnect = true + if isV6 { + config.IPv6Mode = true + } else { + config.IPv4Mode = true + } + getSupportedByIPFamily := func(inputForV4, inputForV6 string) string { // helper to return the current supported single stack IP family (IP/CIDR/IP family name) + if isV6 { + return inputForV6 + } + return inputForV4 + } + getSupportedIPFamilyForDBID := func() egressIPFamilyValue { + if isV6 { + return IPFamilyValueV6 + } + return IPFamilyValueV4 + } + getSupportedOVNIPFamily := func() string { + if isV6 { + return "6" + } + return "4" + } + type nodeInfo struct { // help construct a Node obj / ovn config for each test node + nodeName string + primaryINFIP string + podSubnet string + transitSWIP string + } + // node 1 is index 0, node 2 is index 1, node 3 is index 2 in the slice + nodes := []nodeInfo{{node1Name, getSupportedByIPFamily("192.168.126.12", "fc00:f853:ccd:e793::2"), + getSupportedByIPFamily(v4Node1Subnet, v6Node1Subnet), getSupportedByIPFamily("100.88.0.2", "fd97::2")}, + {node2Name, getSupportedByIPFamily("192.168.126.13", "fc00:f853:ccd:e793::3"), + getSupportedByIPFamily(v4Node2Subnet, v6Node2Subnet), getSupportedByIPFamily("100.88.0.3", "fd97::3")}, + {node3Name, getSupportedByIPFamily("192.168.126.14", "fc00:f853:ccd:e793::4"), + getSupportedByIPFamily(v4Node3Subnet, v6Node3Subnet), getSupportedByIPFamily("100.88.0.4", "fd97::4")}} + mask := "/24" // use same mask for all IP family's since it doesn't matter for this test + + generateNodeObj := func(n nodeInfo) corev1.Node { + // used to populate annotations - supports only a single IP family + addValueToIPFamilyKey := func(value string) string { + key := "ipv4" + if isV6 { + key = "ipv6" + } + return fmt.Sprintf("\"%s\":\"%s\"", key, value) + } + nodeAnnotations := map[string]string{ + "k8s.ovn.org/l3-gateway-config": fmt.Sprintf(`{"default":{"mode":"shared","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"%s", "next-hop":"192.168.126.1"}}`, n.primaryINFIP+mask), + "k8s.ovn.org/node-primary-ifaddr": fmt.Sprintf("{%s}", addValueToIPFamilyKey(n.primaryINFIP+mask)), + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\"]}", n.podSubnet), + "k8s.ovn.org/node-transit-switch-port-ifaddr": fmt.Sprintf("{%s}", addValueToIPFamilyKey(n.transitSWIP+mask)), + "k8s.ovn.org/zone-name": n.nodeName, + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", n.primaryINFIP+mask), + } + return getNodeObj(n.nodeName, nodeAnnotations, map[string]string{}) + } + node1, node2, node3 := generateNodeObj(nodes[0]), generateNodeObj(nodes[1]), generateNodeObj(nodes[2]) + egressNamespace := newNamespace(eipNamespace) + podIP := getSupportedByIPFamily(podV4IP, podV6IP) + egressPod := newPodWithLabels(eipNamespace, podName, node1.Name, podIP, egressPodLabel) + eipIP1 := getSupportedByIPFamily("192.168.126.200", "fc00:f853:ccd:e793::200") + eipIP2 := getSupportedByIPFamily("192.168.126.201", "fc00:f853:ccd:e793::201") + ginkgo.By("creating EgressIP that is assigned to node 2 and node 3") + eIPObj := egressipv1.EgressIP{ + ObjectMeta: newEgressIPMeta(egressIPName), + Spec: egressipv1.EgressIPSpec{ + EgressIPs: []string{ + eipIP1, + eipIP2, + }, + PodSelector: metav1.LabelSelector{ + MatchLabels: egressPodLabel, + }, + NamespaceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": egressNamespace.Name, + }, + }, + }, + Status: egressipv1.EgressIPStatus{ + Items: []egressipv1.EgressIPStatusItem{ + // node 1 was assigned eipIP1 however, it is no longer assigned to this node. The OVN DBs will start + // with node 1 configured with IP eipIP1 and upon sync, the OVN DB must update to reflect + // the state defined here in the EgressIPStatus + { + Node: node2.Name, // remote + EgressIP: eipIP1, + }, + { + Node: node3.Name, // remote + EgressIP: eipIP2, + }, + }, + }, + } + ginkgo.By("start OVN DBs with egress nodes set to node 1 & 2 (EIP status states node 2 & 3)") + getNodeLogicalPortName := func(nodeName string) *string { n := "k8s-" + nodeName; return &n } + // node 1 is a stale next hop. EIP IP moved to node 3. + node1GWToJoinIP := getSupportedByIPFamily(nodeLogicalRouterIPv4[0], nodeLogicalRouterIPv6[0]) + staleHops := []string{node1GWToJoinIP, nodes[1].transitSWIP} // nodes[1] is node 2 + + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.NBGlobal{UUID: "nbglobal-UUID", Name: node1.Name}, + // LRPs to support EIP assigned to local and a remote node + getReRoutePolicy(podIP, getSupportedOVNIPFamily(), "stale-reroute-UUID", + staleHops, getEgressIPLRPReRouteDbIDs(eIPObj.Name, egressPod.Namespace, egressPod.Name, + getSupportedIPFamilyForDBID(), types.DefaultNetworkName, DefaultNetworkControllerName).GetExternalIDs()), + // stale NAT to support EIP that was previously assigned to the local node but OVN DB config persists + &nbdb.NAT{ + UUID: "stale-nat-UUID", + LogicalIP: podIP, + ExternalIP: eipIP1, + ExternalIDs: getEgressIPNATDbIDs(egressIPName, egressPod.Namespace, egressPod.Name, getSupportedIPFamilyForDBID(), + DefaultNetworkControllerName).GetExternalIDs(), + Type: nbdb.NATTypeSNAT, + LogicalPort: getNodeLogicalPortName(node1.Name), + Options: map[string]string{ + "stateless": "false", + }, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name, + Networks: []string{node1GWToJoinIP + mask}, + }, + &nbdb.LogicalRouter{ + Name: types.OVNClusterRouter, + UUID: types.OVNClusterRouter + "-UUID", + Policies: []string{"stale-reroute-UUID"}, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + UUID: types.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, + Nat: []string{"stale-nat-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1.Name + "-UUID", + Name: "k8s-" + node1.Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr( + ovntest.MustParseIPNet(nodes[0].podSubnet)).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1.Name + "-UUID", + Name: node1.Name, + Ports: []string{"k8s-" + node1.Name + "-UUID"}, + }, + }, + }, + &corev1.NamespaceList{ + Items: []corev1.Namespace{*egressNamespace}, + }, + &corev1.PodList{ + Items: []corev1.Pod{*egressPod}, + }, + &corev1.NodeList{ + Items: []corev1.Node{node1, node2, node3}, + }, + &egressipv1.EgressIPList{ + Items: []egressipv1.EgressIP{eIPObj}, + }, + ) + i, podIPv4Net, _ := net.ParseCIDR(podIP + mask) + podIPv4Net.IP = i + fakeOvn.controller.logicalPortCache.add(egressPod, "", types.DefaultNetworkName, "", + nil, []*net.IPNet{podIPv4Net}) + + err := fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIP() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("ensuring removal of invalid local external gateway next hop") + egressIPServedPodsASv4, egressIPServedPodsASv6 := buildEgressIPServedPodsAddressSets([]string{podIP}, + types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName) + egressIPServedPodsAS := egressIPServedPodsASv4 + if isV6 { + egressIPServedPodsAS = egressIPServedPodsASv6 + } + validNextHops := []string{nodes[1].transitSWIP, nodes[2].transitSWIP} // node 2 and 3 + expectedDatabaseState := []libovsdbtest.TestData{ + &nbdb.NBGlobal{UUID: "nbglobal-UUID", Name: node1.Name}, + // LRPs to support EIP assigned to a remote node + // valid LRP for IPv4/IPv6. IPv4 Egress Node is local, IPv6 is remote + getReRoutePolicy(podIP, getSupportedOVNIPFamily(), "valid-reroute-UUID", + validNextHops, getEgressIPLRPReRouteDbIDs(eIPObj.Name, egressPod.Namespace, egressPod.Name, + getSupportedIPFamilyForDBID(), types.DefaultNetworkName, fakeOvn.controller.eIPC.controllerName).GetExternalIDs()), + &nbdb.LogicalRouter{ + Name: types.OVNClusterRouter, + UUID: types.OVNClusterRouter + "-UUID", + Policies: []string{"valid-reroute-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID", + Name: types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name, + Networks: []string{node1GWToJoinIP + mask}, + }, + &nbdb.LogicalRouter{ + Name: types.GWRouterPrefix + node1.Name, + UUID: types.GWRouterPrefix + node1.Name + "-UUID", + Ports: []string{types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + node1.Name + "-UUID"}, + }, + &nbdb.LogicalSwitchPort{ + UUID: "k8s-" + node1.Name + "-UUID", + Name: "k8s-" + node1.Name, + Addresses: []string{"fe:1a:b2:3f:0e:fb " + util.GetNodeManagementIfAddr( + ovntest.MustParseIPNet(nodes[0].podSubnet)).IP.String()}, + }, + &nbdb.LogicalSwitch{ + UUID: node1.Name + "-UUID", + Name: node1.Name, + Ports: []string{"k8s-" + node1.Name + "-UUID"}, + }, + egressIPServedPodsAS, + } + // if IPv6 enabled and IPv4 is disabled, we still create the IPv4 served pods AS + if isV6 { + expectedDatabaseState = append(expectedDatabaseState, egressIPServedPodsASv4) + } + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + ginkgo.By("ensure config is consistent") + gomega.Consistently(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + return nil + } + + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }, ginkgo.Entry("IPv4", false), ginkgo.Entry("IPv6", true)) + ginkgo.It("removes config for previously selected pods on a deleted Node", func() { // node 1 is local zone and egress Node. // pod was on node 2 but it is deleted. Node 2 previously was also an egress Node. @@ -14055,9 +14783,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.localZoneNodes.Store(node1Name, true) fakeOvn.controller.localZoneNodes.Store(node2Name, false) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -14342,9 +15070,9 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations cluster default network" fakeOvn.controller.localZoneNodes.Store(node1Name, true) fakeOvn.controller.localZoneNodes.Store(node2Name, false) - err := fakeOvn.controller.WatchEgressIPNamespaces() + err := fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/egressip_udn_l2_test.go b/go-controller/pkg/ovn/egressip_udn_l2_test.go index 581992ab6b..672fb2143b 100644 --- a/go-controller/pkg/ovn/egressip_udn_l2_test.go +++ b/go-controller/pkg/ovn/egressip_udn_l2_test.go @@ -309,10 +309,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(fakeOvn.networkManager.Start()).Should(gomega.Succeed()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -674,10 +674,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -1173,11 +1173,11 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer fakeOvn.networkManager.Stop() - err = fakeOvn.controller.WatchEgressNodes() + err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1660,15 +1660,15 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer fakeOvn.networkManager.Stop() - err = fakeOvn.controller.WatchEgressNodes() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureRouterPoliciesForNetwork(netInfo, &node1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2030,10 +2030,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -2375,11 +2375,11 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer fakeOvn.networkManager.Stop() - err = fakeOvn.controller.WatchEgressNodes() + err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2741,11 +2741,11 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer fakeOvn.networkManager.Stop() - err = fakeOvn.controller.WatchEgressNodes() + err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPPods() + err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/egressip_udn_l3_test.go b/go-controller/pkg/ovn/egressip_udn_l3_test.go index 28035e8374..c5698b8d08 100644 --- a/go-controller/pkg/ovn/egressip_udn_l3_test.go +++ b/go-controller/pkg/ovn/egressip_udn_l3_test.go @@ -311,10 +311,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(fakeOvn.networkManager.Start()).Should(gomega.Succeed()) defer fakeOvn.networkManager.Stop() - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -684,10 +684,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(secConInfo.bnc.GetNetInfo(), node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -1196,10 +1196,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol err = fakeOvn.networkManager.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer fakeOvn.networkManager.Stop() - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -1941,10 +1941,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -2315,10 +2315,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.eIPController.ensureSwitchPoliciesForNode(netInfo, node1Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP() @@ -2666,10 +2666,10 @@ var _ = ginkgo.Describe("EgressIP Operations for user defined network with topol defer fakeOvn.networkManager.Stop() fakeOvn.controller.zone = node1Name fakeOvn.eIPController.zone = node1Name - err = fakeOvn.controller.WatchEgressIPNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIPPods() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.WatchEgressIPNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressNodes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOvn.controller.WatchEgressIP()