diff --git a/go-controller/pkg/libovsdb/ops/router.go b/go-controller/pkg/libovsdb/ops/router.go index b862095d79..9550b13f2b 100644 --- a/go-controller/pkg/libovsdb/ops/router.go +++ b/go-controller/pkg/libovsdb/ops/router.go @@ -137,6 +137,18 @@ func DeleteLogicalRouter(nbClient libovsdbclient.Client, router *nbdb.LogicalRou // LOGICAL ROUTER PORT OPs +type logicalRouterPortPredicate func(*nbdb.LogicalRouterPort) bool + +// FindLogicalRouterPortWithPredicate looks up logical router port from +// the cache based on a given predicate +func FindLogicalRouterPortWithPredicate(nbClient libovsdbclient.Client, p logicalRouterPortPredicate) ([]*nbdb.LogicalRouterPort, error) { + ctx, cancel := context.WithTimeout(context.Background(), types.OVSDBTimeout) + defer cancel() + found := []*nbdb.LogicalRouterPort{} + err := nbClient.WhereCache(p).List(ctx, &found) + return found, err +} + // GetLogicalRouterPort looks up a logical router port from the cache func GetLogicalRouterPort(nbClient libovsdbclient.Client, lrp *nbdb.LogicalRouterPort) (*nbdb.LogicalRouterPort, error) { found := []*nbdb.LogicalRouterPort{} diff --git a/go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go b/go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go index 11a48e06b5..649725f103 100644 --- a/go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go +++ b/go-controller/pkg/network-attach-def-controller/network_attach_def_controller.go @@ -317,14 +317,21 @@ func (nadController *NetAttachDefinitionController) onNetworkAttachDefinitionUpd } func (nadController *NetAttachDefinitionController) onNetworkAttachDefinitionDelete(obj interface{}) { - nad := obj.(*nettypes.NetworkAttachmentDefinition) - if nad == nil { - utilruntime.HandleError(fmt.Errorf("invalid net-attach-def provided to onNetworkAttachDefinitionDelete()")) - return + nad, ok := obj.(*nettypes.NetworkAttachmentDefinition) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + utilruntime.HandleError(fmt.Errorf("couldn't get object from tombstone %#v", obj)) + return + } + nad, ok = tombstone.Obj.(*nettypes.NetworkAttachmentDefinition) + if !ok { + utilruntime.HandleError(fmt.Errorf("tombstone contained object that is not a NetworkAttachmentDefinition object %#v", obj)) + return + } } - klog.V(4).Infof("%s: Deleting net-attach-def %s/%s", nadController.name, nad.Namespace, nad.Name) - nadController.queueNetworkAttachDefinition(obj) + nadController.queueNetworkAttachDefinition(nad) } // getAllNetworkControllers returns a snapshot of all managed NAD associated network controllers. diff --git a/go-controller/pkg/node/node_ip_handler_linux.go b/go-controller/pkg/node/node_ip_handler_linux.go index 54669461f0..ad3a029dab 100644 --- a/go-controller/pkg/node/node_ip_handler_linux.go +++ b/go-controller/pkg/node/node_ip_handler_linux.go @@ -208,6 +208,8 @@ func (c *addressManager) runInternal(stopChan <-chan struct{}, doneWg *sync.Wait // updates OVN's EncapIP if the node IP changed func (c *addressManager) handleNodePrimaryAddrChange() { + c.Lock() + defer c.Unlock() nodePrimaryAddrChanged, err := c.nodePrimaryAddrChanged() if err != nil { klog.Errorf("Address Manager failed to check node primary address change: %v", err) @@ -215,7 +217,7 @@ func (c *addressManager) handleNodePrimaryAddrChange() { } if nodePrimaryAddrChanged { klog.Infof("Node primary address changed to %v. Updating OVN encap IP.", c.nodePrimaryAddr) - c.updateOVNEncapIPAndReconnect() + updateOVNEncapIPAndReconnect(c.nodePrimaryAddr) } } @@ -334,7 +336,7 @@ func (c *addressManager) nodePrimaryAddrChanged() (bool, error) { if nodePrimaryAddr == nil { return false, fmt.Errorf("failed to parse the primary IP address string from kubernetes node status") } - c.Lock() + var exists bool for _, hostCIDR := range c.cidrs.UnsortedList() { ip, _, err := net.ParseCIDR(hostCIDR) @@ -348,7 +350,6 @@ func (c *addressManager) nodePrimaryAddrChanged() (bool, error) { break } } - c.Unlock() if !exists || c.nodePrimaryAddr.Equal(nodePrimaryAddr) { return false, nil @@ -358,48 +359,6 @@ func (c *addressManager) nodePrimaryAddrChanged() (bool, error) { return true, nil } -// updateOVNEncapIP updates encap IP to OVS when the node primary IP changed. -func (c *addressManager) updateOVNEncapIPAndReconnect() { - checkCmd := []string{ - "get", - "Open_vSwitch", - ".", - "external_ids:ovn-encap-ip", - } - encapIP, stderr, err := util.RunOVSVsctl(checkCmd...) - if err != nil { - klog.Warningf("Unable to retrieve configured ovn-encap-ip from OVS: %v, %q", err, stderr) - } else { - encapIP = strings.TrimSuffix(encapIP, "\n") - if len(encapIP) > 0 && c.nodePrimaryAddr.String() == encapIP { - klog.V(4).Infof("Will not update encap IP, value: %s is the already configured", c.nodePrimaryAddr) - return - } - } - - confCmd := []string{ - "set", - "Open_vSwitch", - ".", - fmt.Sprintf("external_ids:ovn-encap-ip=%s", c.nodePrimaryAddr), - } - - _, stderr, err = util.RunOVSVsctl(confCmd...) - if err != nil { - klog.Errorf("Error setting OVS encap IP: %v %q", err, stderr) - return - } - - // force ovn-controller to reconnect SB with new encap IP immediately. - // otherwise there will be a max delay of 200s due to the 100s - // ovn-controller inactivity probe. - _, stderr, err = util.RunOVNAppctlWithTimeout(5, "-t", "ovn-controller", "exit", "--restart") - if err != nil { - klog.Errorf("Failed to exit ovn-controller %v %q", err, stderr) - return - } -} - // detects if the IP is valid for a node // excludes things like local IPs, mgmt port ip, special masquerade IP func (c *addressManager) isValidNodeIP(addr net.IP) bool { @@ -470,3 +429,45 @@ func (c *addressManager) sync() { c.OnChanged() } } + +// updateOVNEncapIPAndReconnect updates encap IP to OVS when the node primary IP changed. +func updateOVNEncapIPAndReconnect(newIP net.IP) { + checkCmd := []string{ + "get", + "Open_vSwitch", + ".", + "external_ids:ovn-encap-ip", + } + encapIP, stderr, err := util.RunOVSVsctl(checkCmd...) + if err != nil { + klog.Warningf("Unable to retrieve configured ovn-encap-ip from OVS: %v, %q", err, stderr) + } else { + encapIP = strings.TrimSuffix(encapIP, "\n") + if len(encapIP) > 0 && newIP.String() == encapIP { + klog.V(4).Infof("Will not update encap IP %s - it is already configured", newIP.String()) + return + } + } + + confCmd := []string{ + "set", + "Open_vSwitch", + ".", + fmt.Sprintf("external_ids:ovn-encap-ip=%s", newIP), + } + + _, stderr, err = util.RunOVSVsctl(confCmd...) + if err != nil { + klog.Errorf("Error setting OVS encap IP %s: %v %q", newIP.String(), err, stderr) + return + } + + // force ovn-controller to reconnect SB with new encap IP immediately. + // otherwise there will be a max delay of 200s due to the 100s + // ovn-controller inactivity probe. + _, stderr, err = util.RunOVNAppctlWithTimeout(5, "-t", "ovn-controller", "exit", "--restart") + if err != nil { + klog.Errorf("Failed to exit ovn-controller %v %q", err, stderr) + return + } +} diff --git a/go-controller/pkg/ovn/controller/apbroute/network_client.go b/go-controller/pkg/ovn/controller/apbroute/network_client.go index bef8316a11..7a9e4f4b23 100644 --- a/go-controller/pkg/ovn/controller/apbroute/network_client.go +++ b/go-controller/pkg/ovn/controller/apbroute/network_client.go @@ -56,6 +56,14 @@ type conntrackClient struct { podLister corev1listers.PodLister } +func (nb *northBoundClient) findLogicalRouterPortWithPredicate(p func(item *nbdb.LogicalRouterPort) bool) ([]*nbdb.LogicalRouterPort, error) { + return libovsdbops.FindLogicalRouterPortWithPredicate(nb.nbClient, p) +} + +func (nb *northBoundClient) findLogicalRouterPoliciesWithPredicate(p func(item *nbdb.LogicalRouterPolicy) bool) ([]*nbdb.LogicalRouterPolicy, error) { + return libovsdbops.FindLogicalRouterPoliciesWithPredicate(nb.nbClient, p) +} + func (nb *northBoundClient) findLogicalRouterStaticRoutesWithPredicate(p func(item *nbdb.LogicalRouterStaticRoute) bool) ([]*nbdb.LogicalRouterStaticRoute, error) { return libovsdbops.FindLogicalRouterStaticRoutesWithPredicate(nb.nbClient, p) } @@ -464,9 +472,10 @@ func (nb *northBoundClient) deletePodGWRoute(routeInfo *RouteInfo, podIP, gw, gr } node := util.GetWorkerFromGatewayRouter(gr) + // The gw is deleted from the routes cache after this func is called, length 1 // means it is the last gw for the pod and the hybrid route policy should be deleted. - if entry := routeInfo.PodExternalRoutes[podIP]; len(entry) == 1 { + if entry := routeInfo.PodExternalRoutes[podIP]; len(entry) <= 1 { if err := nb.delHybridRoutePolicyForPod(net.ParseIP(podIP), node); err != nil { return fmt.Errorf("unable to delete hybrid route policy for pod %s: err: %v", routeInfo.PodName, err) } @@ -528,72 +537,75 @@ func (nb *northBoundClient) deleteLogicalRouterStaticRoute(podIP, mask, gw, gr s // DelHybridRoutePolicyForPod handles deleting a logical route policy that // forces pod egress traffic to be rerouted to a gateway router for local gateway mode. func (nb *northBoundClient) delHybridRoutePolicyForPod(podIP net.IP, node string) error { - if config.Gateway.Mode == config.GatewayModeLocal { - // Delete podIP from the node's address_set. - asIndex := GetHybridRouteAddrSetDbIDs(node, nb.controllerName) - as, err := nb.addressSetFactory.EnsureAddressSet(asIndex) - if err != nil { - return fmt.Errorf("cannot Ensure that addressSet for node %s exists %v", node, err) + if config.Gateway.Mode != config.GatewayModeLocal { + return nil + } + + // Delete podIP from the node's address_set. + asIndex := GetHybridRouteAddrSetDbIDs(node, nb.controllerName) + as, err := nb.addressSetFactory.EnsureAddressSet(asIndex) + if err != nil { + return fmt.Errorf("cannot Ensure that addressSet for node %s exists %v", node, err) + } + err = as.DeleteIPs([]net.IP{podIP}) + if err != nil { + return fmt.Errorf("unable to remove PodIP %s: to the address set %s, err: %v", podIP.String(), node, err) + } + + // delete hybrid policy to bypass lr-policy in GR, only if there are zero pods on this node. + ipv4HashedAS, ipv6HashedAS := as.GetASHashNames() + ipv4PodIPs, ipv6PodIPs := as.GetIPs() + deletePolicy := false + var l3Prefix string + var matchSrcAS string + if utilnet.IsIPv6(podIP) { + l3Prefix = "ip6" + if len(ipv6PodIPs) == 0 { + deletePolicy = true } - err = as.DeleteIPs([]net.IP{(podIP)}) - if err != nil { - return fmt.Errorf("unable to remove PodIP %s: to the address set %s, err: %v", podIP.String(), node, err) + matchSrcAS = ipv6HashedAS + } else { + l3Prefix = "ip4" + if len(ipv4PodIPs) == 0 { + deletePolicy = true } - - // delete hybrid policy to bypass lr-policy in GR, only if there are zero pods on this node. - ipv4HashedAS, ipv6HashedAS := as.GetASHashNames() - ipv4PodIPs, ipv6PodIPs := as.GetIPs() - deletePolicy := false - var l3Prefix string - var matchSrcAS string - if utilnet.IsIPv6(podIP) { - l3Prefix = "ip6" - if len(ipv6PodIPs) == 0 { - deletePolicy = true + matchSrcAS = ipv4HashedAS + } + if deletePolicy { + var matchDst string + var clusterL3Prefix string + for _, clusterSubnet := range config.Default.ClusterSubnets { + if utilnet.IsIPv6CIDR(clusterSubnet.CIDR) { + clusterL3Prefix = "ip6" + } else { + clusterL3Prefix = "ip4" } - matchSrcAS = ipv6HashedAS - } else { - l3Prefix = "ip4" - if len(ipv4PodIPs) == 0 { - deletePolicy = true + if l3Prefix != clusterL3Prefix { + continue } - matchSrcAS = ipv4HashedAS + matchDst += fmt.Sprintf(" && %s.dst != %s", l3Prefix, clusterSubnet.CIDR) } - if deletePolicy { - var matchDst string - var clusterL3Prefix string - for _, clusterSubnet := range config.Default.ClusterSubnets { - if utilnet.IsIPv6CIDR(clusterSubnet.CIDR) { - clusterL3Prefix = "ip6" - } else { - clusterL3Prefix = "ip4" - } - if l3Prefix != clusterL3Prefix { - continue - } - matchDst += fmt.Sprintf(" && %s.dst != %s", l3Prefix, clusterSubnet.CIDR) - } - matchStr := fmt.Sprintf(`inport == "%s%s" && %s.src == $%s`, types.RouterToSwitchPrefix, node, l3Prefix, matchSrcAS) - matchStr += matchDst + matchStr := fmt.Sprintf(`inport == "%s%s" && %s.src == $%s`, types.RouterToSwitchPrefix, node, l3Prefix, matchSrcAS) + matchStr += matchDst - p := func(item *nbdb.LogicalRouterPolicy) bool { - return item.Priority == types.HybridOverlayReroutePriority && item.Match == matchStr - } - err := libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(nb.nbClient, types.OVNClusterRouter, p) - if err != nil { - return fmt.Errorf("error deleting policy %s on router %s: %v", matchStr, types.OVNClusterRouter, err) - } + p := func(item *nbdb.LogicalRouterPolicy) bool { + return item.Priority == types.HybridOverlayReroutePriority && item.Match == matchStr } - if len(ipv4PodIPs) == 0 && len(ipv6PodIPs) == 0 { - // delete address set. - err := as.Destroy() - if err != nil { - return fmt.Errorf("failed to remove address set: %s, on: %s, err: %v", - as.GetName(), node, err) - } + err := libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(nb.nbClient, types.OVNClusterRouter, p) + if err != nil { + return fmt.Errorf("error deleting policy %s on router %s: %v", matchStr, types.OVNClusterRouter, err) + } + } + if len(ipv4PodIPs) == 0 && len(ipv6PodIPs) == 0 { + // delete address set. + err := as.Destroy() + if err != nil { + return fmt.Errorf("failed to remove address set: %s, on: %s, err: %v", + as.GetName(), node, err) } } return nil + } // extSwitchPrefix returns the prefix of the external switch to use for diff --git a/go-controller/pkg/ovn/controller/apbroute/repair.go b/go-controller/pkg/ovn/controller/apbroute/repair.go index 7a4dc58f99..65f35b2670 100644 --- a/go-controller/pkg/ovn/controller/apbroute/repair.go +++ b/go-controller/pkg/ovn/controller/apbroute/repair.go @@ -157,10 +157,31 @@ func (c *ExternalGatewayMasterController) Repair() error { // if pod had no ECMP routes we need to make sure we remove logical route policy for local gw mode if !podHasAnyECMPRoutes { for _, ovnRoute := range ovnRoutes { - gr := strings.TrimPrefix(ovnRoute.router, types.GWRouterPrefix) - if err := c.nbClient.delHybridRoutePolicyForPod(net.ParseIP(podIP), gr); err != nil { + node := strings.TrimPrefix(ovnRoute.router, types.GWRouterPrefix) + if err := c.nbClient.delHybridRoutePolicyForPod(net.ParseIP(podIP), node); err != nil { return fmt.Errorf("error while removing hybrid policy for pod IP: %s, on node: %s, error: %v", - podIP, gr, err) + podIP, node, err) + } + } + } + } + + // could be stale hybrid policies with stale addresses in the set that had no corresponding OVN ecmp routes + // get all pods, attempt to delete their hybridRoutePolicy that have no policy + if config.Gateway.Mode == config.GatewayModeLocal { + ovnHybridCache, err := c.buildOVNHybridCache() + if err != nil { + return fmt.Errorf("failed to build hybrid cache: %w", err) + } + for ip, node := range ovnHybridCache { + // check if this pod IP has a corresponding policy, if not, remove it + _, okPolicy := policyGWIPsMap[ip] + _, okAnnotation := annotatedGWIPsMap[ip] + if !okPolicy && !okAnnotation { + klog.Infof("CleanHybridPRoutes: Removing IP: %s from hybrid route policy", ip) + if err := c.nbClient.delHybridRoutePolicyForPod(net.ParseIP(ip), node); err != nil { + return fmt.Errorf("CleanHybridPRoutes: error while removing hybrid policy for pod IP: %s, on node: %s, error: %v", + ip, node, err) } } } @@ -184,6 +205,7 @@ func (c *ExternalGatewayMasterController) syncPoliciesWithoutCleanup() (map[stri // because handling them will cause startup failure. // db objects for these policies will be cleaned up, and policies will be handled from scratch by the // general controller logic. + klog.Infof("Skip initial sync for APBRoute policy %s", policy.Name) continue } _, err = c.mgr.syncRoutePolicy(policy.Name) @@ -395,3 +417,63 @@ func (c *ExternalGatewayMasterController) buildOVNECMPCache() (map[string][]*ovn } return ovnRouteCache, nil } + +// returns a hybrid cache of map[podIPs]nodeName +func (c *ExternalGatewayMasterController) buildOVNHybridCache() (map[string]string, error) { + ovnHybridCache := make(map[string]string) + p := func(item *nbdb.LogicalRouterPolicy) bool { + return item.Priority == types.HybridOverlayReroutePriority + } + + logicalRouterPolicies, err := c.nbClient.findLogicalRouterPoliciesWithPredicate(p) + if err != nil { + return nil, fmt.Errorf("CleanHybridPRoutes: failed to list hybrid routes: %v", err) + } + + foundNextHops := sets.Set[string]{} + for _, lrp := range logicalRouterPolicies { + foundNextHops.Insert(lrp.Nexthops...) + } + + r := func(item *nbdb.LogicalRouterPort) bool { + for _, ip := range item.Networks { + // grab only IP prefix and not mask + p := strings.Split(ip, "/") + if foundNextHops.Has(p[0]) { + return true + } + } + return false + } + + grPorts, err := c.nbClient.findLogicalRouterPortWithPredicate(r) + if err != nil { + return nil, fmt.Errorf("CleanHybridPRoutes: failed to search for logical router port: %v", err) + } + + for _, grPort := range grPorts { + nodeName := strings.TrimPrefix(grPort.Name, types.GWRouterToJoinSwitchPrefix+types.GWRouterPrefix) + if len(nodeName) == 0 && nodeName != grPort.Name { + continue + } + + // nodeName has been found + // get address set and list all addresses + asIndex := GetHybridRouteAddrSetDbIDs(nodeName, c.nbClient.controllerName) + as, err := c.nbClient.addressSetFactory.GetAddressSet(asIndex) + if err != nil { + klog.Errorf("CleanHybridPRoutes: unable to find get address set %s: %v", asIndex, err) + continue + } + ipv4Addrs, ipv6Addrs := as.GetIPs() + for _, ip := range ipv4Addrs { + ovnHybridCache[ip] = nodeName + } + for _, ip := range ipv6Addrs { + ovnHybridCache[ip] = nodeName + } + } + + klog.Infof("CleanHybridRoutes: OVN cache built: %#v", ovnHybridCache) + return ovnHybridCache, nil +} diff --git a/go-controller/pkg/ovn/egressgw.go b/go-controller/pkg/ovn/egressgw.go index 9147c8f6d9..2394875064 100644 --- a/go-controller/pkg/ovn/egressgw.go +++ b/go-controller/pkg/ovn/egressgw.go @@ -383,9 +383,10 @@ func (oc *DefaultNetworkController) deletePodGWRoute(routeInfo *apbroutecontroll routeInfo.PodName, gr, gw) node := util.GetWorkerFromGatewayRouter(gr) + // The gw is deleted from the routes cache after this func is called, length 1 // means it is the last gw for the pod and the hybrid route policy should be deleted. - if entry := routeInfo.PodExternalRoutes[podIP]; len(entry) == 1 { + if entry := routeInfo.PodExternalRoutes[podIP]; len(entry) <= 1 { if err := oc.delHybridRoutePolicyForPod(net.ParseIP(podIP), node); err != nil { return fmt.Errorf("unable to delete hybrid route policy for pod %s: err: %v", routeInfo.PodName, err) } @@ -832,7 +833,7 @@ func (oc *DefaultNetworkController) delHybridRoutePolicyForPod(podIP net.IP, nod if err != nil { return fmt.Errorf("cannot Ensure that addressSet for node %s exists %v", node, err) } - err = as.DeleteIPs([]net.IP{(podIP)}) + err = as.DeleteIPs([]net.IP{podIP}) if err != nil { return fmt.Errorf("unable to remove PodIP %s: to the address set %s, err: %v", podIP.String(), node, err) } diff --git a/go-controller/pkg/ovn/egressgw_test.go b/go-controller/pkg/ovn/egressgw_test.go index 9e9fb9e534..e9e555727a 100644 --- a/go-controller/pkg/ovn/egressgw_test.go +++ b/go-controller/pkg/ovn/egressgw_test.go @@ -9,6 +9,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" adminpolicybasedrouteapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/apbroute" @@ -2846,6 +2847,119 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { return nil } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("delete stale addresses from legacy hybrid route policies on startup", func() { + app.Action = func(ctx *cli.Context) error { + config.Gateway.Mode = config.GatewayModeLocal + asIndex := apbroute.GetHybridRouteAddrSetDbIDs("node1", DefaultNetworkControllerName) + asv4, _ := addressset.GetHashNamesForAS(asIndex) + + node1 := tNode{ + Name: "node1", + NodeIP: "1.2.3.4", + NodeLRPMAC: "0a:58:0a:01:01:01", + LrpIP: "100.64.0.2", + LrpIPv6: "fd98::2", + DrLrpIP: "100.64.0.1", + PhysicalBridgeMAC: "11:22:33:44:55:66", + SystemID: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", + NodeSubnet: "10.1.1.0/24", + GWRouter: ovntypes.GWRouterPrefix + "node1", + GatewayRouterIPMask: "172.16.16.2/24", + GatewayRouterIP: "172.16.16.2", + GatewayRouterNextHop: "172.16.16.1", + PhysicalBridgeName: "br-eth0", + NodeGWIP: "10.1.1.1/24", + NodeMgmtPortIP: "10.1.1.2", + NodeMgmtPortMAC: "0a:58:0a:01:01:02", + DnatSnatIP: "169.254.0.1", + } + // create a test node and annotate it with host subnet + testNode := node1.k8sNode("2") + + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.LogicalRouterStaticRoute{ + UUID: "static-route-1-UUID", + IPPrefix: "10.128.1.3/32", + Nexthop: "9.0.0.1", + Options: map[string]string{ + "ecmp_symmetric_reply": "true", + }, + OutputPort: &logicalRouterPort, + Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, + }, + &nbdb.LogicalRouterPolicy{ + UUID: "501-new-UUID", + Priority: ovntypes.HybridOverlayReroutePriority, + Action: nbdb.LogicalRouterPolicyActionReroute, + Nexthops: []string{"100.64.0.4"}, + Match: "inport == \"rtos-node1\" && ip4.src == $" + asv4 + " && ip4.dst != 10.128.0.0/14", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"501-new-UUID"}, + }, + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + StaticRoutes: []string{"static-route-1-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", + Networks: []string{"100.64.0.4/32"}, + }, + }, + }, + &v1.NodeList{ + Items: []v1.Node{ + testNode, + }, + }, + ) + + nodeAnnotator := kube.NewNodeAnnotator(&kube.Kube{KClient: fakeOvn.fakeClient.KubeClient}, + testNode.Name) + + vlanID := uint(1024) + l3Config := node1.gatewayConfig(config.GatewayModeLocal, vlanID) + err := util.SetL3GatewayConfig(nodeAnnotator, l3Config) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = nodeAnnotator.Run() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // add address set with one legit IP that exists in a ecmp route, and one that doesn't + _, err = fakeOvn.asf.NewAddressSet(asIndex, []net.IP{net.ParseIP("10.128.1.3"), net.ParseIP("1.1.1.1")}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + finalNB := []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", + Networks: []string{"100.64.0.4/32"}, + }, + } + + err = fakeOvn.controller.apbExternalRouteController.Repair() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + fakeOvn.asf.EventuallyExpectNoAddressSet(asIndex) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) + + return nil + } + err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) diff --git a/go-controller/pkg/ovn/external_gateway_apb_test.go b/go-controller/pkg/ovn/external_gateway_apb_test.go index 54cd396721..19c17921a9 100644 --- a/go-controller/pkg/ovn/external_gateway_apb_test.go +++ b/go-controller/pkg/ovn/external_gateway_apb_test.go @@ -11,6 +11,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" adminpolicybasedrouteapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1" adminpolicybasedrouteclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/clientset/versioned" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/apbroute" @@ -2698,7 +2699,8 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { injectNode(fakeOvn) fakeOvn.RunAPBExternalPolicyController() - err := fakeOvn.controller.apbExternalRouteController.DelHybridRoutePolicyForPod(net.ParseIP("10.128.1.3"), "node1") + err := fakeOvn.controller.apbExternalRouteController.DelHybridRoutePolicyForPod( + net.ParseIP("10.128.1.3"), "node1") gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) dbIDs := apbroute.GetHybridRouteAddrSetDbIDs("node1", DefaultNetworkControllerName) @@ -2858,6 +2860,119 @@ var _ = ginkgo.Describe("OVN for APB External Route Operations", func() { return nil } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("delete stale addresses from apb hybrid route policies on startup", func() { + app.Action = func(ctx *cli.Context) error { + config.Gateway.Mode = config.GatewayModeLocal + asIndex := apbroute.GetHybridRouteAddrSetDbIDs("node1", DefaultNetworkControllerName) + asv4, _ := addressset.GetHashNamesForAS(asIndex) + + node1 := tNode{ + Name: "node1", + NodeIP: "1.2.3.4", + NodeLRPMAC: "0a:58:0a:01:01:01", + LrpIP: "100.64.0.2", + LrpIPv6: "fd98::2", + DrLrpIP: "100.64.0.1", + PhysicalBridgeMAC: "11:22:33:44:55:66", + SystemID: "cb9ec8fa-b409-4ef3-9f42-d9283c47aac6", + NodeSubnet: "10.1.1.0/24", + GWRouter: ovntypes.GWRouterPrefix + "node1", + GatewayRouterIPMask: "172.16.16.2/24", + GatewayRouterIP: "172.16.16.2", + GatewayRouterNextHop: "172.16.16.1", + PhysicalBridgeName: "br-eth0", + NodeGWIP: "10.1.1.1/24", + NodeMgmtPortIP: "10.1.1.2", + NodeMgmtPortMAC: "0a:58:0a:01:01:02", + DnatSnatIP: "169.254.0.1", + } + // create a test node and annotate it with host subnet + testNode := node1.k8sNode("2") + + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.LogicalRouterStaticRoute{ + UUID: "static-route-1-UUID", + IPPrefix: "10.128.1.3/32", + Nexthop: "9.0.0.1", + Options: map[string]string{ + "ecmp_symmetric_reply": "true", + }, + OutputPort: &logicalRouterPort, + Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, + }, + &nbdb.LogicalRouterPolicy{ + UUID: "501-new-UUID", + Priority: ovntypes.HybridOverlayReroutePriority, + Action: nbdb.LogicalRouterPolicyActionReroute, + Nexthops: []string{"100.64.0.4"}, + Match: "inport == \"rtos-node1\" && ip4.src == $" + asv4 + " && ip4.dst != 10.128.0.0/14", + }, + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + Policies: []string{"501-new-UUID"}, + }, + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + StaticRoutes: []string{"static-route-1-UUID"}, + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", + Networks: []string{"100.64.0.4/32"}, + }, + }, + }, + &v1.NodeList{ + Items: []v1.Node{ + testNode, + }, + }, + ) + + nodeAnnotator := kube.NewNodeAnnotator(&kube.Kube{KClient: fakeOvn.fakeClient.KubeClient}, + testNode.Name) + + vlanID := uint(1024) + l3Config := node1.gatewayConfig(config.GatewayModeLocal, vlanID) + err := util.SetL3GatewayConfig(nodeAnnotator, l3Config) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = nodeAnnotator.Run() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // add address set with one legit IP that exists in a ecmp route, and one that doesn't + _, err = fakeOvn.asf.NewAddressSet(asIndex, []net.IP{net.ParseIP("10.128.1.3"), net.ParseIP("1.1.1.1")}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + finalNB := []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + Name: ovntypes.OVNClusterRouter, + UUID: ovntypes.OVNClusterRouter + "-UUID", + }, + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + }, + &nbdb.LogicalRouterPort{ + UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", + Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", + Networks: []string{"100.64.0.4/32"}, + }, + } + + err = fakeOvn.controller.apbExternalRouteController.Repair() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + fakeOvn.asf.EventuallyExpectNoAddressSet(asIndex) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) + + return nil + } + err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index 78ad925a80..592abf41c8 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -270,11 +270,7 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { fakeOvn.controller.defaultCOPPUUID, err = EnsureDefaultCOPP(fakeOvn.nbClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - nodeAnnotator := kube.NewNodeAnnotator(&kube.KubeOVN{ - Kube: kube.Kube{KClient: fakeOvn.fakeClient.KubeClient}, - ANPClient: fakeOvn.fakeClient.ANPClient, - EIPClient: fakeOvn.fakeClient.EgressIPClient, - EgressFirewallClient: fakeOvn.fakeClient.EgressFirewallClient}, testNode.Name) + nodeAnnotator := kube.NewNodeAnnotator(&kube.Kube{KClient: fakeOvn.fakeClient.KubeClient}, testNode.Name) vlanID := uint(1024) l3Config := node1.gatewayConfig(config.GatewayModeShared, vlanID) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 1c1f72cdae..25f7ce19e2 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -940,181 +940,6 @@ var _ = ginkgo.Describe("test e2e inter-node connectivity between worker nodes", }) }) -// Validate pods can reach a network running in a container's looback address via -// an external gateway running on eth0 of the container without any tunnel encap. -// Next, the test updates the namespace annotation to point to a second container, -// emulating the ext gateway. This test requires shared gateway mode in the job infra. -var _ = ginkgo.Describe("e2e non-vxlan external gateway and update validation", func() { - const ( - svcname string = "multiple-novxlan-externalgw" - ovnNs string = "ovn-kubernetes" - ovnWorkerNode string = "ovn-worker" - ovnContainer string = "ovnkube-node" - gwContainerNameAlt1 string = "gw-novxlan-test-container-alt1" - gwContainerNameAlt2 string = "gw-novxlan-test-container-alt2" - ovnControlNode string = "ovn-control-plane" - ) - var ( - exGWRemoteIpAlt1 string - exGWRemoteIpAlt2 string - ) - f := wrappedTestFramework(svcname) - - // Determine what mode the CI is running in and get relevant endpoint information for the tests - ginkgo.BeforeEach(func() { - exGWRemoteIpAlt1 = "10.249.3.1" - exGWRemoteIpAlt2 = "10.249.4.1" - if IsIPv6Cluster(f.ClientSet) { - exGWRemoteIpAlt1 = "fc00:f853:ccd:e793::1" - exGWRemoteIpAlt2 = "fc00:f853:ccd:e794::1" - } - }) - - ginkgo.AfterEach(func() { - // tear down the containers simulating the gateways - if cid, _ := runCommand(containerRuntime, "ps", "-qaf", fmt.Sprintf("name=%s", gwContainerNameAlt1)); cid != "" { - if _, err := runCommand(containerRuntime, "rm", "-f", gwContainerNameAlt1); err != nil { - framework.Logf("failed to delete the gateway test container %s %v", gwContainerNameAlt1, err) - } - } - if cid, _ := runCommand(containerRuntime, "ps", "-qaf", fmt.Sprintf("name=%s", gwContainerNameAlt2)); cid != "" { - if _, err := runCommand(containerRuntime, "rm", "-f", gwContainerNameAlt2); err != nil { - framework.Logf("failed to delete the gateway test container %s %v", gwContainerNameAlt2, err) - } - } - }) - - ginkgo.It("Should validate connectivity without vxlan before and after updating the namespace annotation to a new external gateway", func() { - - var pingSrc string - var validIP net.IP - - isIPv6Cluster := IsIPv6Cluster(f.ClientSet) - srcPingPodName := "e2e-exgw-novxlan-src-ping-pod" - command := []string{"bash", "-c", "sleep 20000"} - testContainer := fmt.Sprintf("%s-container", srcPingPodName) - testContainerFlag := fmt.Sprintf("--container=%s", testContainer) - // non-ha ci mode runs a set of kind nodes prefixed with ovn-worker - ciWorkerNodeSrc := ovnWorkerNode - - // start the container that will act as an external gateway - _, err := runCommand(containerRuntime, "run", "-itd", "--privileged", "--network", externalContainerNetwork, "--name", gwContainerNameAlt1, agnhostImage) - if err != nil { - framework.Failf("failed to start external gateway test container %s: %v", gwContainerNameAlt1, err) - } - // retrieve the container ip of the external gateway container - alt1IPv4, alt1IPv6 := getContainerAddressesForNetwork(gwContainerNameAlt1, externalContainerNetwork) - if err != nil { - framework.Failf("failed to start external gateway test container: %v", err) - } - nodeIPv4, nodeIPv6 := getContainerAddressesForNetwork(ciWorkerNodeSrc, externalContainerNetwork) - - exGWRemoteCidrAlt1 := fmt.Sprintf("%s/24", exGWRemoteIpAlt1) - exGWRemoteCidrAlt2 := fmt.Sprintf("%s/24", exGWRemoteIpAlt2) - exGWIpAlt1 := alt1IPv4 - nodeIP := nodeIPv4 - if isIPv6Cluster { - exGWIpAlt1 = alt1IPv6 - exGWRemoteCidrAlt1 = fmt.Sprintf("%s/64", exGWRemoteIpAlt1) - exGWRemoteCidrAlt2 = fmt.Sprintf("%s/64", exGWRemoteIpAlt2) - nodeIP = nodeIPv6 - } - - // annotate the test namespace - annotateArgs := []string{ - "annotate", - "namespace", - f.Namespace.Name, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIpAlt1), - } - framework.Logf("Annotating the external gateway test namespace to a container gw: %s ", exGWIpAlt1) - framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) - - podCIDR, err := getNodePodCIDR(ciWorkerNodeSrc) - if err != nil { - framework.Failf("Error retrieving the pod cidr from %s %v", ciWorkerNodeSrc, err) - } - framework.Logf("the pod cidr for node %s is %s", ciWorkerNodeSrc, podCIDR) - // add loopback interface used to validate all traffic is getting drained through the gateway - _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt1, "ip", "address", "add", exGWRemoteCidrAlt1, "dev", "lo") - if err != nil { - framework.Failf("failed to add the loopback ip to dev lo on the test container: %v", err) - } - // Create the pod that will be used as the source for the connectivity test - createGenericPod(f, srcPingPodName, ciWorkerNodeSrc, f.Namespace.Name, command) - // wait for pod setup to return a valid address - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - pingSrc = getPodAddress(srcPingPodName, f.Namespace.Name) - validIP = net.ParseIP(pingSrc) - if validIP == nil { - return false, nil - } - return true, nil - }) - // Fail the test if no address is ever retrieved - if err != nil { - framework.Failf("Error trying to get the pod IP address") - } - // add a host route on the first mock gateway for return traffic to the pod - _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt1, "ip", "route", "add", pingSrc, "via", nodeIP) - if err != nil { - framework.Failf("failed to add the pod host route on the test container: %v", err) - } - _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt1, "ping", "-c", "5", pingSrc) - framework.ExpectNoError(err, "Failed to ping %s from container %s", pingSrc, gwContainerNameAlt1) - - time.Sleep(time.Second * 15) - // Verify the gateway and remote address is reachable from the initial pod - ginkgo.By(fmt.Sprintf("Verifying connectivity without vxlan to the updated annotation and initial external gateway %s and remote address %s", exGWIpAlt1, exGWRemoteIpAlt1)) - _, err = framework.RunKubectl(f.Namespace.Name, "exec", srcPingPodName, testContainerFlag, "--", "ping", "-w", "40", exGWRemoteIpAlt1) - if err != nil { - framework.Failf("Failed to ping the first gateway network %s from container %s on node %s: %v", exGWRemoteIpAlt1, ovnContainer, ovnWorkerNode, err) - } - // start the container that will act as a new external gateway that the tests will be updated to use - _, err = runCommand(containerRuntime, "run", "-itd", "--privileged", "--network", externalContainerNetwork, "--name", gwContainerNameAlt2, agnhostImage) - if err != nil { - framework.Failf("failed to start external gateway test container %s: %v", gwContainerNameAlt2, err) - } - // retrieve the container ip of the external gateway container - alt2IPv4, alt2IPv6 := getContainerAddressesForNetwork(gwContainerNameAlt2, externalContainerNetwork) - exGWIpAlt2 := alt2IPv4 - if isIPv6Cluster { - exGWIpAlt2 = alt2IPv6 - } - - // override the annotation in the test namespace with the new gateway - annotateArgs = []string{ - "annotate", - "namespace", - f.Namespace.Name, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIpAlt2), - "--overwrite", - } - framework.Logf("Annotating the external gateway test namespace to a new container remote IP:%s gw:%s ", exGWIpAlt2, exGWRemoteIpAlt2) - framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) - // add loopback interface used to validate all traffic is getting drained through the gateway - _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt2, "ip", "address", "add", exGWRemoteCidrAlt2, "dev", "lo") - if err != nil { - framework.Failf("failed to add the loopback ip to dev lo on the test container: %v", err) - } - // add a host route on the second mock gateway for return traffic to the pod - _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt2, "ip", "route", "add", pingSrc, "via", nodeIP) - if err != nil { - framework.Failf("failed to add the pod route on the test container: %v", err) - } - - _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt2, "ping", "-c", "5", pingSrc) - framework.ExpectNoError(err, "Failed to ping %s from container %s", pingSrc, gwContainerNameAlt1) - - // Verify the updated gateway and remote address is reachable from the initial pod - ginkgo.By(fmt.Sprintf("Verifying connectivity without vxlan to the updated annotation and new external gateway %s and remote IP %s", exGWRemoteIpAlt2, exGWIpAlt2)) - _, err = framework.RunKubectl(f.Namespace.Name, "exec", srcPingPodName, testContainerFlag, "--", "ping", "-w", "40", exGWRemoteIpAlt2) - if err != nil { - framework.Failf("Failed to ping the second gateway network %s from container %s on node %s: %v", exGWRemoteIpAlt2, ovnContainer, ovnWorkerNode, err) - } - }) -}) - func createSrcPod(podName, nodeName string, ipCheckInterval, ipCheckTimeout time.Duration, f *framework.Framework) { _, err := createGenericPod(f, podName, nodeName, f.Namespace.Name, []string{"bash", "-c", "sleep 20000"}) @@ -2425,140 +2250,6 @@ var _ = ginkgo.Describe("e2e ingress to host-networked pods traffic validation", }) }) -// This test validates ingress traffic sourced from a mock external gateway -// running as a container. Add a namespace annotated with the IP of the -// mock external container's eth0 address. Add a loopback address and a -// route pointing to the pod in the test namespace. Validate connectivity -// sourcing from the mock gateway container loopback to the test ns pod. -var _ = ginkgo.Describe("e2e ingress gateway traffic validation", func() { - const ( - svcname string = "novxlan-externalgw-ingress" - gwContainer string = "gw-ingress-test-container" - ) - - f := wrappedTestFramework(svcname) - - type nodeInfo struct { - name string - nodeIP string - } - - var ( - workerNodeInfo nodeInfo - IsIPv6 bool - ) - - ginkgo.BeforeEach(func() { - - // retrieve worker node names - nodes, err := e2enode.GetBoundedReadySchedulableNodes(f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP) - workerNodeInfo = nodeInfo{ - name: nodes.Items[1].Name, - nodeIP: ips[1], - } - IsIPv6 = IsIPv6Cluster(f.ClientSet) - }) - - ginkgo.AfterEach(func() { - // tear down the container simulating the gateway - if cid, _ := runCommand(containerRuntime, "ps", "-qaf", fmt.Sprintf("name=%s", gwContainer)); cid != "" { - if _, err := runCommand(containerRuntime, "rm", "-f", gwContainer); err != nil { - framework.Logf("failed to delete the gateway test container %s %v", gwContainer, err) - } - } - }) - - ginkgo.It("Should validate ingress connectivity from an external gateway", func() { - - var ( - pingDstPod string - dstPingPodName = "e2e-exgw-ingress-ping-pod" - command = []string{"bash", "-c", "sleep 20000"} - exGWLo = "10.30.1.1" - exGWLoCidr = fmt.Sprintf("%s/32", exGWLo) - pingCmd = ipv4PingCommand - pingCount = "3" - ) - if IsIPv6 { - exGWLo = "fc00::1" // unique local ipv6 unicast addr as per rfc4193 - exGWLoCidr = fmt.Sprintf("%s/64", exGWLo) - pingCmd = ipv6PingCommand - } - - // start the first container that will act as an external gateway - _, err := runCommand(containerRuntime, "run", "-itd", "--privileged", "--network", externalContainerNetwork, - "--name", gwContainer, agnhostImage) - if err != nil { - framework.Failf("failed to start external gateway test container %s: %v", gwContainer, err) - } - exGWIp, exGWIpv6 := getContainerAddressesForNetwork(gwContainer, externalContainerNetwork) - if IsIPv6 { - exGWIp = exGWIpv6 - } - // annotate the test namespace with the external gateway address - annotateArgs := []string{ - "annotate", - "namespace", - f.Namespace.Name, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIp), - } - framework.Logf("Annotating the external gateway test namespace to container gateway: %s", exGWIp) - framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) - - nodeIP, nodeIPv6 := getContainerAddressesForNetwork(workerNodeInfo.name, externalContainerNetwork) - if IsIPv6 { - nodeIP = nodeIPv6 - } - framework.Logf("the pod side node is %s and the source node ip is %s", workerNodeInfo.name, nodeIP) - podCIDR, err := getNodePodCIDR(workerNodeInfo.name) - if err != nil { - framework.Failf("Error retrieving the pod cidr from %s %v", workerNodeInfo.name, err) - } - framework.Logf("the pod cidr for node %s is %s", workerNodeInfo.name, podCIDR) - - // Create the pod that will be used as the source for the connectivity test - createGenericPod(f, dstPingPodName, workerNodeInfo.name, f.Namespace.Name, command) - // wait for the pod setup to return a valid address - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - pingDstPod = getPodAddress(dstPingPodName, f.Namespace.Name) - validIP := net.ParseIP(pingDstPod) - if validIP == nil { - return false, nil - } - return true, nil - }) - // fail the test if a pod address is never retrieved - if err != nil { - framework.Failf("Error trying to get the pod IP address") - } - // add a host route on the gateway for return traffic to the pod - _, err = runCommand(containerRuntime, "exec", gwContainer, "ip", "route", "add", pingDstPod, "via", nodeIP) - if err != nil { - framework.Failf("failed to add the pod host route on the test container %s: %v", gwContainer, err) - } - // add a loopback address to the mock container that will source the ingress test - _, err = runCommand(containerRuntime, "exec", gwContainer, "ip", "address", "add", exGWLoCidr, "dev", "lo") - if err != nil { - framework.Failf("failed to add the loopback ip to dev lo on the test container: %v", err) - } - - // Validate connectivity from the external gateway loopback to the pod in the test namespace - ginkgo.By(fmt.Sprintf("Validate ingress traffic from the external gateway %s can reach the pod in the exgw annotated namespace", gwContainer)) - // generate traffic that will verify connectivity from the mock external gateway loopback - _, err = runCommand(containerRuntime, "exec", gwContainer, string(pingCmd), "-c", pingCount, "-I", "eth0", pingDstPod) - if err != nil { - framework.Failf("failed to ping the pod address %s from mock container %s: %v", pingDstPod, gwContainer, err) - } - }) -}) - // This test validates that OVS exports flow monitoring data from br-int to an external collector var _ = ginkgo.Describe("e2e br-int flow monitoring export validation", func() { type flowMonitoringProtocol string diff --git a/test/e2e/external_gateways.go b/test/e2e/external_gateways.go index c2d1d7fcef..5a01e9cf29 100644 --- a/test/e2e/external_gateways.go +++ b/test/e2e/external_gateways.go @@ -72,6 +72,315 @@ type gatewayTestIPs struct { var _ = ginkgo.Describe("External Gateway", func() { + // Validate pods can reach a network running in a container's looback address via + // an external gateway running on eth0 of the container without any tunnel encap. + // Next, the test updates the namespace annotation to point to a second container, + // emulating the ext gateway. This test requires shared gateway mode in the job infra. + var _ = ginkgo.Describe("e2e non-vxlan external gateway and update validation", func() { + const ( + svcname string = "multiple-novxlan-externalgw" + ovnNs string = "ovn-kubernetes" + ovnWorkerNode string = "ovn-worker" + ovnContainer string = "ovnkube-node" + gwContainerNameAlt1 string = "gw-novxlan-test-container-alt1" + gwContainerNameAlt2 string = "gw-novxlan-test-container-alt2" + ovnControlNode string = "ovn-control-plane" + ) + var ( + exGWRemoteIpAlt1 string + exGWRemoteIpAlt2 string + ) + f := wrappedTestFramework(svcname) + + // Determine what mode the CI is running in and get relevant endpoint information for the tests + ginkgo.BeforeEach(func() { + exGWRemoteIpAlt1 = "10.249.3.1" + exGWRemoteIpAlt2 = "10.249.4.1" + if IsIPv6Cluster(f.ClientSet) { + exGWRemoteIpAlt1 = "fc00:f853:ccd:e793::1" + exGWRemoteIpAlt2 = "fc00:f853:ccd:e794::1" + } + }) + + ginkgo.AfterEach(func() { + // tear down the containers simulating the gateways + if cid, _ := runCommand(containerRuntime, "ps", "-qaf", fmt.Sprintf("name=%s", gwContainerNameAlt1)); cid != "" { + if _, err := runCommand(containerRuntime, "rm", "-f", gwContainerNameAlt1); err != nil { + framework.Logf("failed to delete the gateway test container %s %v", gwContainerNameAlt1, err) + } + } + if cid, _ := runCommand(containerRuntime, "ps", "-qaf", fmt.Sprintf("name=%s", gwContainerNameAlt2)); cid != "" { + if _, err := runCommand(containerRuntime, "rm", "-f", gwContainerNameAlt2); err != nil { + framework.Logf("failed to delete the gateway test container %s %v", gwContainerNameAlt2, err) + } + } + }) + + ginkgo.It("Should validate connectivity without vxlan before and after updating the namespace annotation to a new external gateway", func() { + + var pingSrc string + var validIP net.IP + + isIPv6Cluster := IsIPv6Cluster(f.ClientSet) + srcPingPodName := "e2e-exgw-novxlan-src-ping-pod" + command := []string{"bash", "-c", "sleep 20000"} + testContainer := fmt.Sprintf("%s-container", srcPingPodName) + testContainerFlag := fmt.Sprintf("--container=%s", testContainer) + // non-ha ci mode runs a set of kind nodes prefixed with ovn-worker + ciWorkerNodeSrc := ovnWorkerNode + + // start the container that will act as an external gateway + _, err := runCommand(containerRuntime, "run", "-itd", "--privileged", "--network", externalContainerNetwork, "--name", gwContainerNameAlt1, agnhostImage) + if err != nil { + framework.Failf("failed to start external gateway test container %s: %v", gwContainerNameAlt1, err) + } + // retrieve the container ip of the external gateway container + alt1IPv4, alt1IPv6 := getContainerAddressesForNetwork(gwContainerNameAlt1, externalContainerNetwork) + if err != nil { + framework.Failf("failed to start external gateway test container: %v", err) + } + nodeIPv4, nodeIPv6 := getContainerAddressesForNetwork(ciWorkerNodeSrc, externalContainerNetwork) + + exGWRemoteCidrAlt1 := fmt.Sprintf("%s/24", exGWRemoteIpAlt1) + exGWRemoteCidrAlt2 := fmt.Sprintf("%s/24", exGWRemoteIpAlt2) + exGWIpAlt1 := alt1IPv4 + nodeIP := nodeIPv4 + if isIPv6Cluster { + exGWIpAlt1 = alt1IPv6 + exGWRemoteCidrAlt1 = fmt.Sprintf("%s/64", exGWRemoteIpAlt1) + exGWRemoteCidrAlt2 = fmt.Sprintf("%s/64", exGWRemoteIpAlt2) + nodeIP = nodeIPv6 + } + + // annotate the test namespace + annotateArgs := []string{ + "annotate", + "namespace", + f.Namespace.Name, + fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIpAlt1), + } + framework.Logf("Annotating the external gateway test namespace to a container gw: %s ", exGWIpAlt1) + framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) + + podCIDR, err := getNodePodCIDR(ciWorkerNodeSrc) + if err != nil { + framework.Failf("Error retrieving the pod cidr from %s %v", ciWorkerNodeSrc, err) + } + framework.Logf("the pod cidr for node %s is %s", ciWorkerNodeSrc, podCIDR) + // add loopback interface used to validate all traffic is getting drained through the gateway + _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt1, "ip", "address", "add", exGWRemoteCidrAlt1, "dev", "lo") + if err != nil { + framework.Failf("failed to add the loopback ip to dev lo on the test container: %v", err) + } + // Create the pod that will be used as the source for the connectivity test + createGenericPod(f, srcPingPodName, ciWorkerNodeSrc, f.Namespace.Name, command) + // wait for pod setup to return a valid address + err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { + pingSrc = getPodAddress(srcPingPodName, f.Namespace.Name) + validIP = net.ParseIP(pingSrc) + if validIP == nil { + return false, nil + } + return true, nil + }) + // Fail the test if no address is ever retrieved + if err != nil { + framework.Failf("Error trying to get the pod IP address") + } + // add a host route on the first mock gateway for return traffic to the pod + _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt1, "ip", "route", "add", pingSrc, "via", nodeIP) + if err != nil { + framework.Failf("failed to add the pod host route on the test container: %v", err) + } + _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt1, "ping", "-c", "5", pingSrc) + framework.ExpectNoError(err, "Failed to ping %s from container %s", pingSrc, gwContainerNameAlt1) + + time.Sleep(time.Second * 15) + // Verify the gateway and remote address is reachable from the initial pod + ginkgo.By(fmt.Sprintf("Verifying connectivity without vxlan to the updated annotation and initial external gateway %s and remote address %s", exGWIpAlt1, exGWRemoteIpAlt1)) + _, err = framework.RunKubectl(f.Namespace.Name, "exec", srcPingPodName, testContainerFlag, "--", "ping", "-w", "40", exGWRemoteIpAlt1) + if err != nil { + framework.Failf("Failed to ping the first gateway network %s from container %s on node %s: %v", exGWRemoteIpAlt1, ovnContainer, ovnWorkerNode, err) + } + // start the container that will act as a new external gateway that the tests will be updated to use + _, err = runCommand(containerRuntime, "run", "-itd", "--privileged", "--network", externalContainerNetwork, "--name", gwContainerNameAlt2, agnhostImage) + if err != nil { + framework.Failf("failed to start external gateway test container %s: %v", gwContainerNameAlt2, err) + } + // retrieve the container ip of the external gateway container + alt2IPv4, alt2IPv6 := getContainerAddressesForNetwork(gwContainerNameAlt2, externalContainerNetwork) + exGWIpAlt2 := alt2IPv4 + if isIPv6Cluster { + exGWIpAlt2 = alt2IPv6 + } + + // override the annotation in the test namespace with the new gateway + annotateArgs = []string{ + "annotate", + "namespace", + f.Namespace.Name, + fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIpAlt2), + "--overwrite", + } + framework.Logf("Annotating the external gateway test namespace to a new container remote IP:%s gw:%s ", exGWIpAlt2, exGWRemoteIpAlt2) + framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) + // add loopback interface used to validate all traffic is getting drained through the gateway + _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt2, "ip", "address", "add", exGWRemoteCidrAlt2, "dev", "lo") + if err != nil { + framework.Failf("failed to add the loopback ip to dev lo on the test container: %v", err) + } + // add a host route on the second mock gateway for return traffic to the pod + _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt2, "ip", "route", "add", pingSrc, "via", nodeIP) + if err != nil { + framework.Failf("failed to add the pod route on the test container: %v", err) + } + + _, err = runCommand(containerRuntime, "exec", gwContainerNameAlt2, "ping", "-c", "5", pingSrc) + framework.ExpectNoError(err, "Failed to ping %s from container %s", pingSrc, gwContainerNameAlt1) + + // Verify the updated gateway and remote address is reachable from the initial pod + ginkgo.By(fmt.Sprintf("Verifying connectivity without vxlan to the updated annotation and new external gateway %s and remote IP %s", exGWRemoteIpAlt2, exGWIpAlt2)) + _, err = framework.RunKubectl(f.Namespace.Name, "exec", srcPingPodName, testContainerFlag, "--", "ping", "-w", "40", exGWRemoteIpAlt2) + if err != nil { + framework.Failf("Failed to ping the second gateway network %s from container %s on node %s: %v", exGWRemoteIpAlt2, ovnContainer, ovnWorkerNode, err) + } + }) + }) + + // This test validates ingress traffic sourced from a mock external gateway + // running as a container. Add a namespace annotated with the IP of the + // mock external container's eth0 address. Add a loopback address and a + // route pointing to the pod in the test namespace. Validate connectivity + // sourcing from the mock gateway container loopback to the test ns pod. + var _ = ginkgo.Describe("e2e ingress gateway traffic validation", func() { + const ( + svcname string = "novxlan-externalgw-ingress" + gwContainer string = "gw-ingress-test-container" + ) + + f := wrappedTestFramework(svcname) + + type nodeInfo struct { + name string + nodeIP string + } + + var ( + workerNodeInfo nodeInfo + IsIPv6 bool + ) + + ginkgo.BeforeEach(func() { + + // retrieve worker node names + nodes, err := e2enode.GetBoundedReadySchedulableNodes(f.ClientSet, 3) + framework.ExpectNoError(err) + if len(nodes.Items) < 3 { + framework.Failf( + "Test requires >= 3 Ready nodes, but there are only %v nodes", + len(nodes.Items)) + } + ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP) + workerNodeInfo = nodeInfo{ + name: nodes.Items[1].Name, + nodeIP: ips[1], + } + IsIPv6 = IsIPv6Cluster(f.ClientSet) + }) + + ginkgo.AfterEach(func() { + // tear down the container simulating the gateway + if cid, _ := runCommand(containerRuntime, "ps", "-qaf", fmt.Sprintf("name=%s", gwContainer)); cid != "" { + if _, err := runCommand(containerRuntime, "rm", "-f", gwContainer); err != nil { + framework.Logf("failed to delete the gateway test container %s %v", gwContainer, err) + } + } + }) + + ginkgo.It("Should validate ingress connectivity from an external gateway", func() { + + var ( + pingDstPod string + dstPingPodName = "e2e-exgw-ingress-ping-pod" + command = []string{"bash", "-c", "sleep 20000"} + exGWLo = "10.30.1.1" + exGWLoCidr = fmt.Sprintf("%s/32", exGWLo) + pingCmd = ipv4PingCommand + pingCount = "3" + ) + if IsIPv6 { + exGWLo = "fc00::1" // unique local ipv6 unicast addr as per rfc4193 + exGWLoCidr = fmt.Sprintf("%s/64", exGWLo) + pingCmd = ipv6PingCommand + } + + // start the first container that will act as an external gateway + _, err := runCommand(containerRuntime, "run", "-itd", "--privileged", "--network", externalContainerNetwork, + "--name", gwContainer, agnhostImage) + if err != nil { + framework.Failf("failed to start external gateway test container %s: %v", gwContainer, err) + } + exGWIp, exGWIpv6 := getContainerAddressesForNetwork(gwContainer, externalContainerNetwork) + if IsIPv6 { + exGWIp = exGWIpv6 + } + // annotate the test namespace with the external gateway address + annotateArgs := []string{ + "annotate", + "namespace", + f.Namespace.Name, + fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIp), + } + framework.Logf("Annotating the external gateway test namespace to container gateway: %s", exGWIp) + framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) + + nodeIP, nodeIPv6 := getContainerAddressesForNetwork(workerNodeInfo.name, externalContainerNetwork) + if IsIPv6 { + nodeIP = nodeIPv6 + } + framework.Logf("the pod side node is %s and the source node ip is %s", workerNodeInfo.name, nodeIP) + podCIDR, err := getNodePodCIDR(workerNodeInfo.name) + if err != nil { + framework.Failf("Error retrieving the pod cidr from %s %v", workerNodeInfo.name, err) + } + framework.Logf("the pod cidr for node %s is %s", workerNodeInfo.name, podCIDR) + + // Create the pod that will be used as the source for the connectivity test + createGenericPod(f, dstPingPodName, workerNodeInfo.name, f.Namespace.Name, command) + // wait for the pod setup to return a valid address + err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { + pingDstPod = getPodAddress(dstPingPodName, f.Namespace.Name) + validIP := net.ParseIP(pingDstPod) + if validIP == nil { + return false, nil + } + return true, nil + }) + // fail the test if a pod address is never retrieved + if err != nil { + framework.Failf("Error trying to get the pod IP address") + } + // add a host route on the gateway for return traffic to the pod + _, err = runCommand(containerRuntime, "exec", gwContainer, "ip", "route", "add", pingDstPod, "via", nodeIP) + if err != nil { + framework.Failf("failed to add the pod host route on the test container %s: %v", gwContainer, err) + } + // add a loopback address to the mock container that will source the ingress test + _, err = runCommand(containerRuntime, "exec", gwContainer, "ip", "address", "add", exGWLoCidr, "dev", "lo") + if err != nil { + framework.Failf("failed to add the loopback ip to dev lo on the test container: %v", err) + } + + // Validate connectivity from the external gateway loopback to the pod in the test namespace + ginkgo.By(fmt.Sprintf("Validate ingress traffic from the external gateway %s can reach the pod in the exgw annotated namespace", gwContainer)) + // generate traffic that will verify connectivity from the mock external gateway loopback + _, err = runCommand(containerRuntime, "exec", gwContainer, string(pingCmd), "-c", pingCount, "-I", "eth0", pingDstPod) + if err != nil { + framework.Failf("failed to ping the pod address %s from mock container %s: %v", pingDstPod, gwContainer, err) + } + }) + }) + var _ = ginkgo.Context("With annotations", func() { // Validate pods can reach a network running in a container's looback address via