diff --git a/go-controller/pkg/node/gateway.go b/go-controller/pkg/node/gateway.go index ef519c5def..665562ac4b 100644 --- a/go-controller/pkg/node/gateway.go +++ b/go-controller/pkg/node/gateway.go @@ -182,7 +182,7 @@ func (g *gateway) Run(stopChan <-chan struct{}, wg *sync.WaitGroup) { } } -func gatewayInitInternal(nodeName, gwIntf, egressGatewayIntf string, subnets []*net.IPNet, gwNextHops []net.IP, gwIPs []*net.IPNet, nodeAnnotator kube.Annotator) ( +func gatewayInitInternal(nodeName, gwIntf, egressGatewayIntf string, gwNextHops []net.IP, gwIPs []*net.IPNet, nodeAnnotator kube.Annotator) ( *bridgeConfiguration, *bridgeConfiguration, error) { gatewayBridge, err := bridgeForInterface(gwIntf, nodeName, types.PhysicalNetworkName, gwIPs) if err != nil { diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index 0bd45f236c..cb0b7256e8 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -308,11 +308,11 @@ func (n *OvnNode) initGateway(subnets []*net.IPNet, nodeAnnotator kube.Annotator switch config.Gateway.Mode { case config.GatewayModeLocal: klog.Info("Preparing Local Gateway") - gw, err = newLocalGateway(n.name, subnets, gatewayNextHops, gatewayIntf, ifAddrs, nodeAnnotator, + gw, err = newLocalGateway(n.name, subnets, gatewayNextHops, gatewayIntf, egressGWInterface, ifAddrs, nodeAnnotator, managementPortConfig, n.Kube, n.watchFactory) case config.GatewayModeShared: klog.Info("Preparing Shared Gateway") - gw, err = newSharedGateway(n.name, subnets, gatewayNextHops, gatewayIntf, egressGWInterface, ifAddrs, nodeAnnotator, n.Kube, + gw, err = newSharedGateway(n.name, gatewayNextHops, gatewayIntf, egressGWInterface, ifAddrs, nodeAnnotator, n.Kube, managementPortConfig, n.watchFactory) case config.GatewayModeDisabled: var chassisID string diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 1f6f4d98d3..f14b297f51 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -187,7 +187,7 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, defer GinkgoRecover() gatewayNextHops, gatewayIntf, err := getGatewayNextHops() - sharedGw, err := newSharedGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, gatewayIntf, "", nil, nodeAnnotator, k, + sharedGw, err := newSharedGateway(nodeName, gatewayNextHops, gatewayIntf, "", nil, nodeAnnotator, k, &fakeMgmtPortConfig, wf) Expect(err).NotTo(HaveOccurred()) err = sharedGw.Init(wf) @@ -357,7 +357,7 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, Cmd: "ovs-vsctl --timeout=15 get Interface " + hostRep + " Name", Output: hostRep, }) - // newSharedGatewayOpenFlowManager + // newGatewayOpenFlowManager fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ovs-vsctl --timeout=15 get Interface patch-" + brphys + "_node1-to-br-int ofport", Output: "5", @@ -379,7 +379,7 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, Cmd: "ovs-vsctl --timeout=15 get Interface pf0hpf Name", Output: hostRep, }) - // newSharedGatewayOpenFlowManager + // newGatewayOpenFlowManager fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ovs-vsctl --timeout=15 get interface " + hostRep + " ofport", Output: "9", @@ -463,8 +463,8 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, gatewayNextHops, gatewayIntf, err := getGatewayNextHops() // provide host IP as GR IP gwIPs := []*net.IPNet{ovntest.MustParseIPNet(hostCIDR)} - sharedGw, err := newSharedGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, - gatewayIntf, "", gwIPs, nodeAnnotator, k, &fakeMgmtPortConfig, nil) + sharedGw, err := newSharedGateway(nodeName, gatewayNextHops, + gatewayIntf, "", gwIPs, nodeAnnotator, k, &fakeMgmtPortConfig, wf) Expect(err).NotTo(HaveOccurred()) err = sharedGw.Init(wf) @@ -771,7 +771,7 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0`, defer GinkgoRecover() gatewayNextHops, gatewayIntf, err := getGatewayNextHops() - localGw, err := newLocalGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, gatewayIntf, nil, + localGw, err := newLocalGateway(nodeName, ovntest.MustParseIPNets(nodeSubnet), gatewayNextHops, gatewayIntf, "", nil, nodeAnnotator, &fakeMgmtPortConfig, k, wf) Expect(err).NotTo(HaveOccurred()) err = localGw.Init(wf) diff --git a/go-controller/pkg/node/gateway_localnet.go b/go-controller/pkg/node/gateway_localnet.go index ed7384cfde..3aaed5178b 100644 --- a/go-controller/pkg/node/gateway_localnet.go +++ b/go-controller/pkg/node/gateway_localnet.go @@ -5,23 +5,21 @@ package node import ( "fmt" - "net" - "strings" - "sync" - "github.com/coreos/go-iptables/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + "net" + "strings" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" ) -func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net.IP, gwIntf string, - gwIPs []*net.IPNet, nodeAnnotator kube.Annotator, cfg *managementPortConfig, kube kube.Interface, watchFactory factory.NodeWatchFactory) (*gateway, error) { +func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net.IP, gwIntf, egressGWIntf string, gwIPs []*net.IPNet, + nodeAnnotator kube.Annotator, cfg *managementPortConfig, kube kube.Interface, watchFactory factory.NodeWatchFactory) (*gateway, error) { klog.Info("Creating new local gateway") gw := &gateway{} @@ -43,8 +41,8 @@ func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net } } - gwBridge, _, err := gatewayInitInternal( - nodeName, gwIntf, "", hostSubnets, gwNextHops, nil, nodeAnnotator) + gwBridge, exGwBridge, err := gatewayInitInternal( + nodeName, gwIntf, egressGWIntf, gwNextHops, gwIPs, nodeAnnotator) if err != nil { return nil, err } @@ -70,8 +68,22 @@ func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net } // END OCP HACK - gw.readyFunc = func() (bool, error) { - return gatewayReady(gwBridge.patchPort) + if exGwBridge != nil { + gw.readyFunc = func() (bool, error) { + ready, err := gatewayReady(gwBridge.patchPort) + if err != nil { + return false, err + } + exGWReady, err := gatewayReady(exGwBridge.patchPort) + if err != nil { + return false, err + } + return ready && exGWReady, nil + } + } else { + gw.readyFunc = func() (bool, error) { + return gatewayReady(gwBridge.patchPort) + } } gw.initFunc = func() error { @@ -80,7 +92,16 @@ func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net if err != nil { return err } - gw.openflowManager, err = newLocalGatewayOpenflowManager(gwBridge) + if exGwBridge != nil { + err = setBridgeOfPorts(exGwBridge) + if err != nil { + return err + } + } + + gw.nodeIPManager = newAddressManager(nodeName, kube, cfg, watchFactory) + + gw.openflowManager, err = newGatewayOpenFlowManager(gwBridge, exGwBridge) if err != nil { return err } @@ -152,35 +173,3 @@ func cleanupLocalnetGateway(physnet string) error { } return err } - -// since we share the host's k8s node IP, add OpenFlow flows -// -- to steer the NodePort traffic arriving on the host to the OVN logical topology and -// -- to also connection track the outbound north-south traffic through l3 gateway so that -// the return traffic can be steered back to OVN logical topology -// -- to handle host -> service access, via masquerading from the host to OVN GR -// -- to handle external -> service(ExternalTrafficPolicy: Local) -> host access without SNAT -func newLocalGatewayOpenflowManager(gwBridge *bridgeConfiguration) (*openflowManager, error) { - var dftFlows []string - - dftFlows, err := flowsForDefaultBridge(gwBridge.ofPortPhys, gwBridge.macAddress.String(), gwBridge.ofPortPatch, - gwBridge.ofPortHost, gwBridge.ips) - if err != nil { - return nil, err - } - - dftCommonFlows := commonFlows(gwBridge.ofPortPhys, gwBridge.macAddress.String(), gwBridge.ofPortPatch, - gwBridge.ofPortHost) - dftFlows = append(dftFlows, dftCommonFlows...) - - // add health check function to check default OpenFlow flows are on the shared gateway bridge - ofm := &openflowManager{ - defaultBridge: gwBridge, - flowCache: make(map[string][]string), - flowMutex: sync.Mutex{}, - flowChan: make(chan struct{}, 1), - } - ofm.updateFlowCacheEntry("NORMAL", []string{fmt.Sprintf("table=0,priority=0,actions=%s\n", util.NormalAction)}) - ofm.updateFlowCacheEntry("DEFAULT", dftFlows) - ofm.requestFlowSync() - return ofm, nil -} diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index e8d2250ce5..d3e63842ba 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -758,7 +758,7 @@ func (npwipt *nodePortWatcherIptables) SyncServices(services []interface{}) { // the return traffic can be steered back to OVN logical topology // -- to handle host -> service access, via masquerading from the host to OVN GR // -- to handle external -> service(ExternalTrafficPolicy: Local) -> host access without SNAT -func newSharedGatewayOpenFlowManager(gwBridge, exGWBridge *bridgeConfiguration) (*openflowManager, error) { +func newGatewayOpenFlowManager(gwBridge, exGWBridge *bridgeConfiguration) (*openflowManager, error) { dftFlows, err := flowsForDefaultBridge(gwBridge.ofPortPhys, gwBridge.macAddress.String(), gwBridge.ofPortPatch, gwBridge.ofPortHost, gwBridge.ips) if err != nil { @@ -1160,13 +1160,13 @@ func setBridgeOfPorts(bridge *bridgeConfiguration) error { return nil } -func newSharedGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP, gwIntf, egressGWIntf string, +func newSharedGateway(nodeName string, gwNextHops []net.IP, gwIntf, egressGWIntf string, gwIPs []*net.IPNet, nodeAnnotator kube.Annotator, kube kube.Interface, cfg *managementPortConfig, watchFactory factory.NodeWatchFactory) (*gateway, error) { klog.Info("Creating new shared gateway") gw := &gateway{} gwBridge, exGwBridge, err := gatewayInitInternal( - nodeName, gwIntf, egressGWIntf, subnets, gwNextHops, gwIPs, nodeAnnotator) + nodeName, gwIntf, egressGWIntf, gwNextHops, gwIPs, nodeAnnotator) if err != nil { return nil, err } @@ -1224,7 +1224,9 @@ func newSharedGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP return err } } - gw.openflowManager, err = newSharedGatewayOpenFlowManager(gwBridge, exGwBridge) + + gw.nodeIPManager = newAddressManager(nodeName, kube, cfg, watchFactory) + gw.openflowManager, err = newGatewayOpenFlowManager(gwBridge, exGwBridge) if err != nil { return err } diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index b9c9570475..b7c427896e 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -548,7 +548,7 @@ func (oc *Controller) addExternalSwitch(prefix, interfaceID, nodeName, gatewayRo // Connect GR to external_switch with mac address of external interface // and that IP address. In the case of `local` gateway mode, whenever ovnkube-node container // restarts a new br-local bridge will be created with a new `nicMacAddress`. - externalRouterPort := types.GWRouterToExtSwitchPrefix + gatewayRouter + externalRouterPort := prefix + types.GWRouterToExtSwitchPrefix + gatewayRouter externalRouterPortNetworks := []string{} for _, ip := range ipAddresses { @@ -590,7 +590,7 @@ func (oc *Controller) addExternalSwitch(prefix, interfaceID, nodeName, gatewayRo } // Connect the external_switch to the router. - externalSwitchPortToRouter := types.EXTSwitchToGWRouterPrefix + gatewayRouter + externalSwitchPortToRouter := prefix + types.EXTSwitchToGWRouterPrefix + gatewayRouter externalLogicalSwitchPortToRouter := nbdb.LogicalSwitchPort{ Name: externalSwitchPortToRouter,