Bug 1880591: Allow local no bridge

go-controller/pkg/node/OCP_HACKS.go

@@ -3,10 +3,17 @@
 package node
 
 import (
+	"fmt"
+	"net"
+
 	"github.com/coreos/go-iptables/iptables"
+
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 )
 
-// OCP HACK: Block MCS Access. https://github.com/openshift/ovn-kubernetes/pull/170
+// Block MCS Access. https://github.com/openshift/ovn-kubernetes/pull/170
 func generateBlockMCSRules(rules *[]iptRule, protocol iptables.Protocol) {
 	*rules = append(*rules, iptRule{
 		table:    "filter",
@@ -34,4 +41,38 @@ func generateBlockMCSRules(rules *[]iptRule, protocol iptables.Protocol) {
 	})
 }
 
-// END OCP HACK
+// initSharedGatewayNoBridge is used in order to run local gateway mode without moving the NIC to an ovs bridge
+// https://github.com/openshift/ovn-kubernetes/pull/281
+func (n *OvnNode) initSharedGatewayNoBridge(subnets []*net.IPNet, gwNextHops []net.IP, nodeAnnotator kube.Annotator) (postWaitFunc, error) {
+	err := setupLocalNodeAccessBridge(n.name, subnets)
+	if err != nil {
+		return nil, err
+	}
+	chassisID, err := util.GetNodeChassisID()
+	if err != nil {
+		return nil, err
+	}
+	// get the real default interface
+	defaultGatewayIntf, _, err := getDefaultGatewayInterfaceDetails()
+	if err != nil {
+		return nil, err
+	}
+	ips, err := getNetworkInterfaceIPAddresses(defaultGatewayIntf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get interface details for %s (%v)",
+			defaultGatewayIntf, err)
+	}
+	err = util.SetL3GatewayConfig(nodeAnnotator, &util.L3GatewayConfig{
+		ChassisID:      chassisID,
+		Mode:           config.GatewayModeLocal,
+		IPAddresses:    ips,
+		MACAddress:     util.IPAddrToHWAddr(ips[0].IP),
+		NextHops:       gwNextHops,
+		NodePortEnable: config.Gateway.NodeportEnable,
+	})
+	if err != nil {
+		return nil, err
+	} else {
+		return func() error { return nil }, nil
+	}
+}

go-controller/pkg/node/gateway_shared_intf.go

@@ -522,7 +522,13 @@ func (n *OvnNode) initSharedGateway(subnets []*net.IPNet, gwNextHops []net.IP, g
 	var brCreated bool
 	var err error
 
-	if bridgeName, _, err = util.RunOVSVsctl("--", "port-to-br", gwIntf); err == nil {
+	// OCP HACK
+	// Do not configure OVS bridge for local gateway mode with a gateway iface of none
+	// For 4.5->4.6 migration, see 	https://github.com/openshift/ovn-kubernetes/pull/281
+	if gwIntf == "none" {
+		return n.initSharedGatewayNoBridge(subnets, gwNextHops, nodeAnnotator)
+		// END OCP HACK
+	} else if bridgeName, _, err = util.RunOVSVsctl("--", "port-to-br", gwIntf); err == nil {
 		// This is an OVS bridge's internal port
 		uplinkName, err = util.GetNicName(bridgeName)
 		if err != nil {

go-controller/pkg/ovn/OCP_HACKS.go

@@ -0,0 +1,97 @@
+package ovn
+
+import (
+	"fmt"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
+	kapi "k8s.io/api/core/v1"
+	"strings"
+)
+
+// isGatewayInterfaceNone is used to determine if this is a local gateway mode with a "none" gateway interface
+// this indicates if we are in an upgrade mode from 4.5->4.6 where the GR would not exist.
+// See https://github.com/openshift/ovn-kubernetes/pull/281
+func isGatewayInterfaceNone() bool {
+	return config.Gateway.Interface == "none"
+}
+
+// createNodePortLoadBalancers is just a copy of the current node balancer code that will not be invoked during
+// local gateway mode + gateway-interface of "none". So we need to still create them ourselves on for the node
+// switches (since they will not be created on the GR)
+// See https://github.com/openshift/ovn-kubernetes/pull/281
+func createNodePortLoadBalancers(gatewayRouter, nodeName string, sctpSupport bool) error {
+	// Create 3 load-balancers for north-south traffic for each gateway
+	// router: UDP, TCP, SCTP
+	k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP, err := getGatewayLoadBalancers(gatewayRouter)
+	if err != nil {
+		return err
+	}
+	protoLBMap := map[kapi.Protocol]string{
+		kapi.ProtocolTCP:  k8sNSLbTCP,
+		kapi.ProtocolUDP:  k8sNSLbUDP,
+		kapi.ProtocolSCTP: k8sNSLbSCTP,
+	}
+	enabledProtos := []kapi.Protocol{kapi.ProtocolTCP, kapi.ProtocolUDP}
+	if sctpSupport {
+		enabledProtos = append(enabledProtos, kapi.ProtocolSCTP)
+	}
+	var stdout, stderr string
+	for _, proto := range enabledProtos {
+		if protoLBMap[proto] == "" {
+			protoLBMap[proto], stderr, err = util.RunOVNNbctl("--", "create",
+				"load_balancer",
+				fmt.Sprintf("external_ids:%s_lb_gateway_router=%s", proto, gatewayRouter),
+				fmt.Sprintf("protocol=%s", strings.ToLower(string(proto))))
+			if err != nil {
+				return fmt.Errorf("failed to create load balancer for gateway router %s for protocol %s: "+
+					"stderr: %q, error: %v", gatewayRouter, proto, stderr, err)
+			}
+		}
+	}
+
+	// Local gateway mode does not use GR for ingress node port traffic, it uses mp0 instead
+	if config.Gateway.Mode != config.GatewayModeLocal {
+		// Add north-south load-balancers to the gateway router.
+		lbString := fmt.Sprintf("%s,%s", protoLBMap[kapi.ProtocolTCP], protoLBMap[kapi.ProtocolUDP])
+		if sctpSupport {
+			lbString = lbString + "," + protoLBMap[kapi.ProtocolSCTP]
+		}
+		stdout, stderr, err = util.RunOVNNbctl("set", "logical_router", gatewayRouter, "load_balancer="+lbString)
+		if err != nil {
+			return fmt.Errorf("failed to set north-south load-balancers to the "+
+				"gateway router %s, stdout: %q, stderr: %q, error: %v",
+				gatewayRouter, stdout, stderr, err)
+		}
+	}
+	// Also add north-south load-balancers to local switches for pod -> nodePort traffic
+	stdout, stderr, err = util.RunOVNNbctl("get", "logical_switch", nodeName, "load_balancer")
+	if err != nil {
+		return fmt.Errorf("failed to get load-balancers on the node switch %s, stdout: %q, "+
+			"stderr: %q, error: %v", nodeName, stdout, stderr, err)
+	}
+	for _, proto := range enabledProtos {
+		if !strings.Contains(stdout, protoLBMap[proto]) {
+			stdout, stderr, err = util.RunOVNNbctl("ls-lb-add", nodeName, protoLBMap[proto])
+			if err != nil {
+				return fmt.Errorf("failed to add north-south load-balancer %s to the "+
+					"node switch %s, stdout: %q, stderr: %q, error: %v",
+					protoLBMap[proto], nodeName, stdout, stderr, err)
+			}
+		}
+	}
+	return nil
+}
+
+// gatewayInitMinimal sets up the minimal configuration needed for N/S traffic to work in Local gateway mode. In this
+// case we do not need any GR or join switch, just nodePort load balancers on the node switch
+// See https://github.com/openshift/ovn-kubernetes/pull/281
+func gatewayInitMinimal(nodeName string, l3GatewayConfig *util.L3GatewayConfig, sctpSupport bool) error {
+	gatewayRouter := gwRouterPrefix + nodeName
+	if l3GatewayConfig.NodePortEnable {
+		err := createNodePortLoadBalancers(gatewayRouter, nodeName, sctpSupport)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

go-controller/pkg/ovn/endpoints.go

@@ -114,9 +114,14 @@ func (ovn *Controller) AddEndpoints(ep *kapi.Endpoints) error {
 func (ovn *Controller) handleNodePortLB(node *kapi.Node) error {
 	gatewayRouter := gwRouterPrefix + node.Name
 	var physicalIPs []string
-	if physicalIPs, _ = ovn.getGatewayPhysicalIPs(gatewayRouter); physicalIPs == nil {
-		return fmt.Errorf("gateway physical IP for node %q does not yet exist", node.Name)
+	// OCP HACK - there will not be a GR during local gw + no gw interface mode (upgrade from 4.5->4.6)
+	// See https://github.com/openshift/ovn-kubernetes/pull/281
+	if !isGatewayInterfaceNone() {
+		if physicalIPs, _ = ovn.getGatewayPhysicalIPs(gatewayRouter); physicalIPs == nil {
+			return fmt.Errorf("gateway physical IP for node %q does not yet exist", node.Name)
+		}
 	}
+	// END OCP HACK
 	namespaces, err := ovn.watchFactory.GetNamespaces()
 	if err != nil {
 		return fmt.Errorf("failed to get k8s namespaces: %v", err)

go-controller/pkg/ovn/master.go

@@ -449,9 +449,21 @@ func (oc *Controller) syncGatewayLogicalNetwork(node *kapi.Node, l3GatewayConfig
 		return err
 	}
 
-	err = gatewayInit(node.Name, clusterSubnets, hostSubnets, joinSubnets, l3GatewayConfig, oc.SCTPSupport)
-	if err != nil {
-		return fmt.Errorf("failed to init shared interface gateway: %v", err)
+	// OCP HACK
+	// GatewayModeLocal is only used if Local mode is specified and None shared gateway bridge is specified
+	// This is to allow local gateway mode without having to configure/use the shared gateway bridge
+	// See https://github.com/openshift/ovn-kubernetes/pull/281
+	if l3GatewayConfig.Mode == config.GatewayModeLocal {
+		err = gatewayInitMinimal(node.Name, l3GatewayConfig, oc.SCTPSupport)
+		if err != nil {
+			return fmt.Errorf("failed to init local gateway with no OVS bridge: %v", err)
+		}
+		// END OCP HACK
+	} else {
+		err = gatewayInit(node.Name, clusterSubnets, hostSubnets, joinSubnets, l3GatewayConfig, oc.SCTPSupport)
+		if err != nil {
+			return fmt.Errorf("failed to init shared interface gateway: %v", err)
+		}
 	}
 
 	// in the case of shared gateway mode, we need to setup