diff --git a/go-controller/pkg/node/OCP_HACKS.go b/go-controller/pkg/node/OCP_HACKS.go index 903725e156..0de3496780 100644 --- a/go-controller/pkg/node/OCP_HACKS.go +++ b/go-controller/pkg/node/OCP_HACKS.go @@ -3,10 +3,17 @@ package node import ( + "fmt" + "net" + "github.com/coreos/go-iptables/iptables" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) -// OCP HACK: Block MCS Access. https://github.com/openshift/ovn-kubernetes/pull/170 +// Block MCS Access. https://github.com/openshift/ovn-kubernetes/pull/170 func generateBlockMCSRules(rules *[]iptRule, protocol iptables.Protocol) { *rules = append(*rules, iptRule{ table: "filter", @@ -34,4 +41,38 @@ func generateBlockMCSRules(rules *[]iptRule, protocol iptables.Protocol) { }) } -// END OCP HACK +// initSharedGatewayNoBridge is used in order to run local gateway mode without moving the NIC to an ovs bridge +// https://github.com/openshift/ovn-kubernetes/pull/281 +func (n *OvnNode) initSharedGatewayNoBridge(subnets []*net.IPNet, gwNextHops []net.IP, nodeAnnotator kube.Annotator) (postWaitFunc, error) { + err := setupLocalNodeAccessBridge(n.name, subnets) + if err != nil { + return nil, err + } + chassisID, err := util.GetNodeChassisID() + if err != nil { + return nil, err + } + // get the real default interface + defaultGatewayIntf, _, err := getDefaultGatewayInterfaceDetails() + if err != nil { + return nil, err + } + ips, err := getNetworkInterfaceIPAddresses(defaultGatewayIntf) + if err != nil { + return nil, fmt.Errorf("failed to get interface details for %s (%v)", + defaultGatewayIntf, err) + } + err = util.SetL3GatewayConfig(nodeAnnotator, &util.L3GatewayConfig{ + ChassisID: chassisID, + Mode: config.GatewayModeLocal, + IPAddresses: ips, + MACAddress: util.IPAddrToHWAddr(ips[0].IP), + NextHops: gwNextHops, + NodePortEnable: config.Gateway.NodeportEnable, + }) + if err != nil { + return nil, err + } else { + return func() error { return nil }, nil + } +} diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index 53ca299d72..d15e889eb1 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -522,7 +522,13 @@ func (n *OvnNode) initSharedGateway(subnets []*net.IPNet, gwNextHops []net.IP, g var brCreated bool var err error - if bridgeName, _, err = util.RunOVSVsctl("--", "port-to-br", gwIntf); err == nil { + // OCP HACK + // Do not configure OVS bridge for local gateway mode with a gateway iface of none + // For 4.5->4.6 migration, see https://github.com/openshift/ovn-kubernetes/pull/281 + if gwIntf == "none" { + return n.initSharedGatewayNoBridge(subnets, gwNextHops, nodeAnnotator) + // END OCP HACK + } else if bridgeName, _, err = util.RunOVSVsctl("--", "port-to-br", gwIntf); err == nil { // This is an OVS bridge's internal port uplinkName, err = util.GetNicName(bridgeName) if err != nil { diff --git a/go-controller/pkg/ovn/OCP_HACKS.go b/go-controller/pkg/ovn/OCP_HACKS.go new file mode 100644 index 0000000000..4b470c1a7e --- /dev/null +++ b/go-controller/pkg/ovn/OCP_HACKS.go @@ -0,0 +1,97 @@ +package ovn + +import ( + "fmt" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + kapi "k8s.io/api/core/v1" + "strings" +) + +// isGatewayInterfaceNone is used to determine if this is a local gateway mode with a "none" gateway interface +// this indicates if we are in an upgrade mode from 4.5->4.6 where the GR would not exist. +// See https://github.com/openshift/ovn-kubernetes/pull/281 +func isGatewayInterfaceNone() bool { + return config.Gateway.Interface == "none" +} + +// createNodePortLoadBalancers is just a copy of the current node balancer code that will not be invoked during +// local gateway mode + gateway-interface of "none". So we need to still create them ourselves on for the node +// switches (since they will not be created on the GR) +// See https://github.com/openshift/ovn-kubernetes/pull/281 +func createNodePortLoadBalancers(gatewayRouter, nodeName string, sctpSupport bool) error { + // Create 3 load-balancers for north-south traffic for each gateway + // router: UDP, TCP, SCTP + k8sNSLbTCP, k8sNSLbUDP, k8sNSLbSCTP, err := getGatewayLoadBalancers(gatewayRouter) + if err != nil { + return err + } + protoLBMap := map[kapi.Protocol]string{ + kapi.ProtocolTCP: k8sNSLbTCP, + kapi.ProtocolUDP: k8sNSLbUDP, + kapi.ProtocolSCTP: k8sNSLbSCTP, + } + enabledProtos := []kapi.Protocol{kapi.ProtocolTCP, kapi.ProtocolUDP} + if sctpSupport { + enabledProtos = append(enabledProtos, kapi.ProtocolSCTP) + } + var stdout, stderr string + for _, proto := range enabledProtos { + if protoLBMap[proto] == "" { + protoLBMap[proto], stderr, err = util.RunOVNNbctl("--", "create", + "load_balancer", + fmt.Sprintf("external_ids:%s_lb_gateway_router=%s", proto, gatewayRouter), + fmt.Sprintf("protocol=%s", strings.ToLower(string(proto)))) + if err != nil { + return fmt.Errorf("failed to create load balancer for gateway router %s for protocol %s: "+ + "stderr: %q, error: %v", gatewayRouter, proto, stderr, err) + } + } + } + + // Local gateway mode does not use GR for ingress node port traffic, it uses mp0 instead + if config.Gateway.Mode != config.GatewayModeLocal { + // Add north-south load-balancers to the gateway router. + lbString := fmt.Sprintf("%s,%s", protoLBMap[kapi.ProtocolTCP], protoLBMap[kapi.ProtocolUDP]) + if sctpSupport { + lbString = lbString + "," + protoLBMap[kapi.ProtocolSCTP] + } + stdout, stderr, err = util.RunOVNNbctl("set", "logical_router", gatewayRouter, "load_balancer="+lbString) + if err != nil { + return fmt.Errorf("failed to set north-south load-balancers to the "+ + "gateway router %s, stdout: %q, stderr: %q, error: %v", + gatewayRouter, stdout, stderr, err) + } + } + // Also add north-south load-balancers to local switches for pod -> nodePort traffic + stdout, stderr, err = util.RunOVNNbctl("get", "logical_switch", nodeName, "load_balancer") + if err != nil { + return fmt.Errorf("failed to get load-balancers on the node switch %s, stdout: %q, "+ + "stderr: %q, error: %v", nodeName, stdout, stderr, err) + } + for _, proto := range enabledProtos { + if !strings.Contains(stdout, protoLBMap[proto]) { + stdout, stderr, err = util.RunOVNNbctl("ls-lb-add", nodeName, protoLBMap[proto]) + if err != nil { + return fmt.Errorf("failed to add north-south load-balancer %s to the "+ + "node switch %s, stdout: %q, stderr: %q, error: %v", + protoLBMap[proto], nodeName, stdout, stderr, err) + } + } + } + return nil +} + +// gatewayInitMinimal sets up the minimal configuration needed for N/S traffic to work in Local gateway mode. In this +// case we do not need any GR or join switch, just nodePort load balancers on the node switch +// See https://github.com/openshift/ovn-kubernetes/pull/281 +func gatewayInitMinimal(nodeName string, l3GatewayConfig *util.L3GatewayConfig, sctpSupport bool) error { + gatewayRouter := gwRouterPrefix + nodeName + if l3GatewayConfig.NodePortEnable { + err := createNodePortLoadBalancers(gatewayRouter, nodeName, sctpSupport) + if err != nil { + return err + } + } + return nil +} diff --git a/go-controller/pkg/ovn/endpoints.go b/go-controller/pkg/ovn/endpoints.go index 1b79541ce3..9b251caff0 100644 --- a/go-controller/pkg/ovn/endpoints.go +++ b/go-controller/pkg/ovn/endpoints.go @@ -114,9 +114,14 @@ func (ovn *Controller) AddEndpoints(ep *kapi.Endpoints) error { func (ovn *Controller) handleNodePortLB(node *kapi.Node) error { gatewayRouter := gwRouterPrefix + node.Name var physicalIPs []string - if physicalIPs, _ = ovn.getGatewayPhysicalIPs(gatewayRouter); physicalIPs == nil { - return fmt.Errorf("gateway physical IP for node %q does not yet exist", node.Name) + // OCP HACK - there will not be a GR during local gw + no gw interface mode (upgrade from 4.5->4.6) + // See https://github.com/openshift/ovn-kubernetes/pull/281 + if !isGatewayInterfaceNone() { + if physicalIPs, _ = ovn.getGatewayPhysicalIPs(gatewayRouter); physicalIPs == nil { + return fmt.Errorf("gateway physical IP for node %q does not yet exist", node.Name) + } } + // END OCP HACK namespaces, err := ovn.watchFactory.GetNamespaces() if err != nil { return fmt.Errorf("failed to get k8s namespaces: %v", err) diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 3146e4ecd9..6fc4a21292 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -449,9 +449,21 @@ func (oc *Controller) syncGatewayLogicalNetwork(node *kapi.Node, l3GatewayConfig return err } - err = gatewayInit(node.Name, clusterSubnets, hostSubnets, joinSubnets, l3GatewayConfig, oc.SCTPSupport) - if err != nil { - return fmt.Errorf("failed to init shared interface gateway: %v", err) + // OCP HACK + // GatewayModeLocal is only used if Local mode is specified and None shared gateway bridge is specified + // This is to allow local gateway mode without having to configure/use the shared gateway bridge + // See https://github.com/openshift/ovn-kubernetes/pull/281 + if l3GatewayConfig.Mode == config.GatewayModeLocal { + err = gatewayInitMinimal(node.Name, l3GatewayConfig, oc.SCTPSupport) + if err != nil { + return fmt.Errorf("failed to init local gateway with no OVS bridge: %v", err) + } + // END OCP HACK + } else { + err = gatewayInit(node.Name, clusterSubnets, hostSubnets, joinSubnets, l3GatewayConfig, oc.SCTPSupport) + if err != nil { + return fmt.Errorf("failed to init shared interface gateway: %v", err) + } } // in the case of shared gateway mode, we need to setup