[WIP] DNM Allow local hack

.github/workflows/test.yml

@@ -179,13 +179,12 @@ jobs:
          #   ENABLED   v4  ha     shared
          #   ENABLED   v4  noha   local
          #   DISABLED  v4  noha   shared
-         #   DISABLED  v6  ha     local
+         #   ENABLED   v6  ha     local
          #   ENABLED   v6  ha     shared
          #   DISABLED  v6  noha   local
          #   DISABLED  v6  noha   shared
          - {"ipfamily": {"ip": ipv4}, "ha": {"enabled": "true"}, "gateway-mode": local}
          - {"ipfamily": {"ip": ipv4}, "ha": {"enabled": "false"}, "gateway-mode": shared}
-         - {"ipfamily": {"ip": ipv6}, "ha": {"enabled": "true"}, "gateway-mode": local}
          - {"ipfamily": {"ip": ipv6}, "ha": {"enabled": "false"}, "gateway-mode": local}
          - {"ipfamily": {"ip": ipv6}, "ha": {"enabled": "false"}, "gateway-mode": shared}
     needs: [build, k8s]

contrib/kind.sh

@@ -316,7 +316,9 @@ if [ "$OVN_IMAGE" == local ]; then
 
   # Build ovn kube image
   pushd ../dist/images
-  sudo cp -f ../../go-controller/_output/go/bin/* .
+  # Find all built executables, but ignore the 'windows' directory if it exists
+  BINS=$(find ../../go-controller/_output/go/bin/ -maxdepth 1 -type f | xargs)
+  sudo cp -f ${BINS} .
   echo "ref: $(git rev-parse  --symbolic-full-name HEAD)  commit: $(git rev-parse  HEAD)" > git_info
   docker build -t ovn-daemonset-f:dev -f Dockerfile.fedora .
   OVN_IMAGE=ovn-daemonset-f:dev
@@ -356,7 +358,7 @@ pushd ../dist/yaml
 run_kubectl apply -f k8s.ovn.org_egressfirewalls.yaml
 run_kubectl apply -f k8s.ovn.org_egressips.yaml
 run_kubectl apply -f ovn-setup.yaml
-MASTER_NODES=$(kind get nodes --name ${KIND_CLUSTER_NAME} | head -n ${KIND_NUM_MASTER})
+MASTER_NODES=$(kind get nodes --name ${KIND_CLUSTER_NAME} | sort | head -n ${KIND_NUM_MASTER})
 # We want OVN HA not Kubernetes HA
 # leverage the kubeadm well-known label node-role.kubernetes.io/master=
 # to choose the nodes where ovn master components will be placed

dist/images/Dockerfile.ubuntu

@@ -8,11 +8,11 @@
 #
 # So this file will change over time.
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 USER root
 
-RUN apt-get update && apt-get install -y arping iproute2 curl software-properties-common setpriv
+RUN apt-get update && apt-get install -y arping iproute2 curl software-properties-common util-linux
 
 RUN echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list
 RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -

go-controller/pkg/config/config.go

@@ -1113,6 +1113,9 @@ func buildGatewayConfig(ctx *cli.Context, cli, file *config) error {
 	// HACK force local gateway mode
 	Gateway.Mode = GatewayModeLocal
 
+	// HACK force gateway interface none
+	Gateway.Interface = "none"
+
 	return nil
 }
 

go-controller/pkg/factory/factory.go

@@ -441,7 +441,7 @@ type ObjectCacheInterface interface {
 var _ ObjectCacheInterface = &WatchFactory{}
 
 const (
-	resyncInterval        = 12 * time.Hour
+	resyncInterval        = 0
 	handlerAlive   uint32 = 0
 	handlerDead    uint32 = 1
 	numEventQueues int    = 15
@@ -461,11 +461,12 @@ var (
 
 // NewWatchFactory initializes a new watch factory
 func NewWatchFactory(c kubernetes.Interface, eip egressipclientset.Interface, ec egressfirewallclientset.Interface, crd apiextensionsclientset.Interface) (*WatchFactory, error) {
-	// resync time is 12 hours, none of the resources being watched in ovn-kubernetes have
+	// resync time is 0, none of the resources being watched in ovn-kubernetes have
 	// any race condition where a resync may be required e.g. cni executable on node watching for
 	// events on pods and assuming that an 'ADD' event will contain the annotations put in by
 	// ovnkube master (currently, it is just a 'get' loop)
 	// the downside of making it tight (like 10 minutes) is needless spinning on all resources
+	// However, AddEventHandlerWithResyncPeriod can specify a per handler resync period
 	wf := &WatchFactory{
 		iFactory:    informerfactory.NewSharedInformerFactory(c, resyncInterval),
 		eipFactory:  egressipinformerfactory.NewSharedInformerFactory(eip, resyncInterval),

go-controller/pkg/informer/const.go

@@ -1,15 +1,14 @@
 package informer
 
-import "time"
-
 // These constants can be removed at some point
 // They are here to provide backwards-compatibility with the
 // pkg/factory code which provided defaults.
 // The package consumer should make these decisions instead.
 const (
 	// DefaultResyncInterval is the default interval that all caches should
-	// periodically resync
-	DefaultResyncInterval = time.Hour * 12
+	// periodically resync. AddEventHandlerWithResyncPeriod can specify a
+	// per handler resync period if necessary
+	DefaultResyncInterval = 0
 	// DefaultNodeInformerThreadiness is the number of worker routines spawned
 	// to services the Node event queue
 	DefaultNodeInformerThreadiness = 10

go-controller/pkg/node/gateway_init.go

@@ -73,30 +73,23 @@ func bridgedGatewayNodeSetup(nodeName, bridgeName, bridgeInterface, physicalNetw
 
 // getNetworkInterfaceIPAddresses returns the IP addresses for the network interface 'iface'.
 func getNetworkInterfaceIPAddresses(iface string) ([]*net.IPNet, error) {
-	intf, err := net.InterfaceByName(iface)
+	allIPs, err := util.GetNetworkInterfaceIPs(iface)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not find IP addresses: %v", err)
 	}
 
-	addrs, err := intf.Addrs()
-	if err != nil {
-		return nil, err
-	}
 	var ips []*net.IPNet
 	var foundIPv4 bool
 	var foundIPv6 bool
-	for _, addr := range addrs {
-		switch ip := addr.(type) {
-		case *net.IPNet:
-			if utilnet.IsIPv6CIDR(ip) {
-				if config.IPv6Mode && !foundIPv6 {
-					ips = append(ips, ip)
-					foundIPv6 = true
-				}
-			} else if config.IPv4Mode && !foundIPv4 {
+	for _, ip := range allIPs {
+		if utilnet.IsIPv6CIDR(ip) {
+			if config.IPv6Mode && !foundIPv6 {
 				ips = append(ips, ip)
-				foundIPv4 = true
+				foundIPv6 = true
 			}
+		} else if config.IPv4Mode && !foundIPv4 {
+			ips = append(ips, ip)
+			foundIPv4 = true
 		}
 	}
 	if config.IPv4Mode && !foundIPv4 {

go-controller/pkg/node/gateway_shared_intf.go

@@ -522,7 +522,41 @@ func (n *OvnNode) initSharedGateway(subnets []*net.IPNet, gwNextHops []net.IP, g
 	var brCreated bool
 	var err error
 
-	if bridgeName, _, err = util.RunOVSVsctl("--", "port-to-br", gwIntf); err == nil {
+	// OCP HACK
+	if gwIntf == "none" {
+		err = setupLocalNodeAccessBridge(n.name, subnets)
+		if err != nil {
+			return nil, err
+		}
+		chassisID, err := util.GetNodeChassisID()
+		if err != nil {
+			return nil, err
+		}
+		// get the real default interface
+		defaultGatewayIntf, _, err := getDefaultGatewayInterfaceDetails()
+		if err != nil {
+			return nil, err
+		}
+		ips, err := getNetworkInterfaceIPAddresses(defaultGatewayIntf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get interface details for %s (%v)",
+				gwIntf, err)
+		}
+		err = util.SetL3GatewayConfig(nodeAnnotator, &util.L3GatewayConfig{
+			ChassisID:      chassisID,
+			Mode:           config.GatewayModeLocal,
+			IPAddresses:    ips,
+			MACAddress:     util.IPAddrToHWAddr(ips[0].IP),
+			NextHops:       gwNextHops,
+			NodePortEnable: config.Gateway.NodeportEnable,
+		})
+		if err != nil {
+			return nil, err
+		} else {
+			return func() error { return nil }, nil
+		}
+		// END OCP HACK
+	} else if bridgeName, _, err = util.RunOVSVsctl("--", "port-to-br", gwIntf); err == nil {
 		// This is an OVS bridge's internal port
 		uplinkName, err = util.GetNicName(bridgeName)
 		if err != nil {
@@ -586,7 +620,7 @@ func (n *OvnNode) initSharedGateway(subnets []*net.IPNet, gwNextHops []net.IP, g
 
 	return func() error {
 		if config.Gateway.NodeportEnable {
-			if config.Gateway.Mode == config.GatewayModeLocal {
+			if config.Gateway.Mode == config.GatewayModeLocal && gwIntf != "None" {
 				if err := addDefaultConntrackRulesLocal(n.name, bridgeName, uplinkName, n.stopChan); err != nil {
 					return err
 				}

go-controller/pkg/node/gateway_shared_intf_linux.go

@@ -38,7 +38,13 @@ func setupLocalNodeAccessBridge(nodeName string, subnets []*net.IPNet) error {
 		return err
 	}
 
-	macAddress := util.IPAddrToHWAddr(net.ParseIP(util.V4NodeLocalNatSubnetNextHop)).String()
+	var macAddress string
+	if config.IPv4Mode {
+		macAddress = util.IPAddrToHWAddr(net.ParseIP(util.V4NodeLocalNatSubnetNextHop)).String()
+	} else {
+		macAddress = util.IPAddrToHWAddr(net.ParseIP(util.V6NodeLocalNatSubnetNextHop)).String()
+	}
+
 	_, stderr, err = util.RunOVSVsctl(
 		"--may-exist", "add-port", localBridgeName, localnetGatewayNextHopPort,
 		"--", "set", "interface", localnetGatewayNextHopPort, "type=internal",

go-controller/pkg/ovn/egressfirewall.go

@@ -3,8 +3,10 @@ package ovn
 import (
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	egressfirewallapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 
@@ -15,7 +17,6 @@ import (
 )
 
 const (
-	defaultStartPriority           = 2000
 	egressFirewallAppliedCorrectly = "EgressFirewall Rules applied"
 	egressFirewallAddError         = "EgressFirewall Rules not correctly added"
 	egressFirewallUpdateError      = "EgressFirewall Rules did not update correctly"
@@ -81,8 +82,20 @@ func (oc *Controller) addEgressFirewall(egressFirewall *egressfirewallapi.Egress
 	ef := newEgressFirewall(egressFirewall)
 	nsInfo.egressFirewallPolicy = ef
 	var errList []error
+	egressFirewallStartPriorityInt, err := strconv.Atoi(egressFirewallStartPriority)
+	if err != nil {
+		return []error{fmt.Errorf("failed to convert egressFirewallStartPriority to Integer: cannot add egressFirewall for namespace %s", egressFirewall.Namespace)}
+	}
+	minimumReservedEgressFirewallPriorityInt, err := strconv.Atoi(minimumReservedEgressFirewallPriority)
+	if err != nil {
+		return []error{fmt.Errorf("failed to convert minumumReservedEgressFirewallPriority to Integer: cannot add egressFirewall for namespace %s", egressFirewall.Namespace)}
+	}
 	for i, egressFirewallRule := range egressFirewall.Spec.Egress {
 		//process Rules into egressFirewallRules for egressFirewall struct
+		if i > egressFirewallStartPriorityInt-minimumReservedEgressFirewallPriorityInt {
+			klog.Warningf("egressFirewall for namespace %s has to many rules, the rest will be ignored", egressFirewall.Namespace)
+			break
+		}
 		efr, err := newEgressFirewallRule(egressFirewallRule, i)
 		if err != nil {
 			errList = append(errList, fmt.Errorf("error: cannot create EgressFirewall Rule for destination %s to namespace %s - %v",
@@ -96,15 +109,7 @@ func (oc *Controller) addEgressFirewall(egressFirewall *egressfirewallapi.Egress
 		return errList
 	}
 
-	existingNodes, err := oc.kube.GetNodes()
-	if err != nil {
-		return []error{fmt.Errorf("error unable to add egressfirewall %s, cannot list nodes: %s", egressFirewall.Name, err)}
-	}
-	var joinSwitches []string
-	for _, node := range existingNodes.Items {
-		joinSwitches = append(joinSwitches, joinSwitch(node.Name))
-	}
-	err = ef.addACLToJoinSwitch(joinSwitches, nsInfo.addressSet.GetIPv4HashName(), nsInfo.addressSet.GetIPv6HashName())
+	err = ef.addLogicalRouterPolicyToClusterRouter(nsInfo.addressSet.GetIPv4HashName(), nsInfo.addressSet.GetIPv6HashName(), egressFirewallStartPriorityInt)
 	if err != nil {
 		errList = append(errList, err)
 	}
@@ -131,29 +136,19 @@ func (oc *Controller) deleteEgressFirewall(egressFirewall *egressfirewallapi.Egr
 		nsInfo.egressFirewallPolicy = nil
 		nsInfo.Unlock()
 	}
-
-	existingNodes, err := oc.kube.GetNodes()
+	stdout, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--columns=_uuid", "find", "logical_router_policy", fmt.Sprintf("external-ids:egressFirewall=%s", egressFirewall.Namespace))
 	if err != nil {
-		return []error{fmt.Errorf("error deleting egressFirewall for namespace %s, cannot get nodes to delete ACLS %v",
-			egressFirewall.Namespace, err)}
-	}
-
-	stdout, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading",
-		"--columns=_uuid", "find", "ACL",
-		fmt.Sprintf("external-ids:egressFirewall=%s", egressFirewall.Namespace))
-	if err != nil {
-		return []error{fmt.Errorf("error executing find ACL command, stderr: %q, %+v", stderr, err)}
+		return []error{fmt.Errorf("error deleting egressFirewall for namespace %s, cannot get logical router policies from LR %s - %s:%s",
+			egressFirewall.Namespace, ovnClusterRouter, err, stderr)}
 	}
 	var errList []error
+
 	uuids := strings.Fields(stdout)
 	for _, uuid := range uuids {
-		for _, node := range existingNodes.Items {
-			_, stderr, err := util.RunOVNNbctl("remove", "logical_switch",
-				joinSwitch(node.Name), "acls", uuid)
-			if err != nil {
-				errList = append(errList, fmt.Errorf("failed to delete the rules for "+
-					"egressFirewall in namespace %s on node %s, stderr: %q (%v)", egressFirewall.Namespace, node.Name, stderr, err))
-			}
+		_, stderr, err := util.RunOVNNbctl("lr-policy-del", ovnClusterRouter, uuid)
+		if err != nil {
+			errList = append(errList, fmt.Errorf("failed to delete the rules for "+
+				"egressFirewall in namespace %s on logical switch %s, stderr: %q (%v)", egressFirewall.Namespace, ovnClusterRouter, stderr, err))
 		}
 	}
 	return errList
@@ -165,7 +160,7 @@ func (oc *Controller) updateEgressFirewallWithRetry(egressfirewall *egressfirewa
 	})
 }
 
-func (ef *egressFirewall) addACLToJoinSwitch(joinSwitches []string, hashedAddressSetNameIPv4, hashedAddressSetNameIPv6 string) error {
+func (ef *egressFirewall) addLogicalRouterPolicyToClusterRouter(hashedAddressSetNameIPv4, hashedAddressSetNameIPv6 string, efStartPriority int) error {
 	for _, rule := range ef.egressRules {
 		var match string
 		var action string
@@ -180,38 +175,27 @@ func (ef *egressFirewall) addACLToJoinSwitch(joinSwitches []string, hashedAddres
 			return fmt.Errorf("error rule.to.cidrSelector %s is not a valid CIDR (%+v)", rule.to.cidrSelector, err)
 		}
 		if !utilnet.IsIPv6(ipAddress) {
-			match = fmt.Sprintf("match=\"ip4.dst == %s && ip4.src == $%s\"", rule.to.cidrSelector, hashedAddressSetNameIPv4)
+			match = fmt.Sprintf("match=\"ip4.dst == %s && ip4.src == $%s", rule.to.cidrSelector, hashedAddressSetNameIPv4)
 		} else {
-			match = fmt.Sprintf("match=\"ip6.dst == %s && ip6.src == $%s\"", rule.to.cidrSelector, hashedAddressSetNameIPv6)
+			match = fmt.Sprintf("match=\"ip6.dst == %s && ip6.src == $%s", rule.to.cidrSelector, hashedAddressSetNameIPv6)
 		}
 
 		if len(rule.ports) > 0 {
-			match = fmt.Sprintf("%s && ( %s )\"", match[:len(match)-1], egressGetL4Match(rule.ports))
+			match = fmt.Sprintf("%s && ( %s )", match, egressGetL4Match(rule.ports))
 		}
-		uuid, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading",
-			"--columns=_uuid", "find", "ACL", match, "action="+action,
-			fmt.Sprintf("external-ids:egressFirewall=%s", ef.namespace))
 
-		if err != nil {
-			return fmt.Errorf("error executing find ACL command, stderr: %q, %+v", stderr, err)
-		}
-		for _, joinSwitch := range joinSwitches {
-			if uuid == "" {
-				_, stderr, err := util.RunOVNNbctl("--id=@acl", "create", "acl",
-					fmt.Sprintf("priority=%d", defaultStartPriority-rule.id),
-					fmt.Sprintf("direction=%s", fromLport), match, "action="+action,
-					fmt.Sprintf("external-ids:egressFirewall=%s", ef.namespace),
-					"--", "add", "logical_switch", joinSwitch,
-					"acls", "@acl")
-				if err != nil {
-					return fmt.Errorf("error executing create ACL command, stderr: %q, %+v", stderr, err)
-				}
-			} else {
-				_, stderr, err := util.RunOVNNbctl("add", "logical_switch", joinSwitch, "acls", uuid)
-				if err != nil {
-					return fmt.Errorf("error adding ACL to joinsSwitch %s failed, stderr: %q, %+v", joinSwitch, stderr, err)
+		match = fmt.Sprintf("%s && %s\"", match, getClusterSubnetsExclusion())
 
-				}
+		_, stderr, err := util.RunOVNNbctl("--id=@logical_router_policy", "create", "logical_router_policy",
+			fmt.Sprintf("priority=%d", efStartPriority-rule.id),
+			match, "action="+action, fmt.Sprintf("external-ids:egressFirewall=%s", ef.namespace),
+			"--", "add", "logical_router", ovnClusterRouter, "policies", "@logical_router_policy")
+		if err != nil {
+			// TODO: lr-policy-add doesn't support --may-exist, resort to this workaround for now.
+			// Have raised an issue against ovn repository (https://github.com/ovn-org/ovn/issues/49)
+			if !strings.Contains(stderr, "already existed") {
+				return fmt.Errorf("failed to add policy route '%s' to %s "+
+					"stderr: %s, error: %v", match, ovnClusterRouter, stderr, err)
 			}
 		}
 	}
@@ -281,6 +265,14 @@ func egressGetL4Match(ports []egressfirewallapi.EgressFirewallPort) string {
 	return l4Match
 }
 
-func joinSwitch(nodeName string) string {
-	return fmt.Sprintf("join_%s", nodeName)
+func getClusterSubnetsExclusion() string {
+	var exclusion string
+	for _, clusterSubnet := range config.Default.ClusterSubnets {
+		if utilnet.IsIPv6CIDR(clusterSubnet.CIDR) {
+			exclusion += fmt.Sprintf(" && %s.dst != %s", "ip6", clusterSubnet.CIDR)
+		} else {
+			exclusion += fmt.Sprintf(" && %s.dst != %s", "ip4", clusterSubnet.CIDR)
+		}
+	}
+	return exclusion[4:]
 }